Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions admin/templates/download_events/download_events.html
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@
color: #777;
margin-bottom: 8px;
}
.card-sublabel {
font-size: 12px;
color: #9c9c9c;
margin: -4px 0 10px;
}
.card-value {
font-size: 28px;
font-weight: 600;
Expand Down Expand Up @@ -109,6 +114,11 @@ <h2>Download telemetry dashboard</h2>
<div class="dashboard-grid charts-grid">
<div class="dashboard-card chart-card">
<div class="card-label">GB by storage region</div>
<div class="card-sublabel">
Requests in range — Files: {{ download_events_dashboard.split.file.count }}
· Zips: {{ download_events_dashboard.split.zip.count }}
· Total: {{ download_events_dashboard.summary.total_downloads }}
</div>

<div class="chart-container">
<canvas id="storageRegionChart"></canvas>
Expand All @@ -117,6 +127,11 @@ <h2>Download telemetry dashboard</h2>

<div class="dashboard-card chart-card">
<div class="card-label">Downloads / GB by user region</div>
<div class="card-sublabel">
Requests in range — Files: {{ download_events_dashboard.split.file.count }}
· Zips: {{ download_events_dashboard.split.zip.count }}
· Total: {{ download_events_dashboard.summary.total_downloads }}
</div>

<div class="chart-container">
<canvas id="userRegionChart"></canvas>
Expand Down Expand Up @@ -296,6 +311,24 @@ <h4 style="margin:0 0 12px;color:#aaa;text-align:center;">
{% endfor %}
];

const storageFileCounts = [
{% for region in download_events_dashboard.storage_regions %}
{{ region.file_count }},
{% endfor %}
];

const storageZipCounts = [
{% for region in download_events_dashboard.storage_regions %}
{{ region.zip_count }},
{% endfor %}
];

const storageTotalCounts = [
{% for region in download_events_dashboard.storage_regions %}
{{ region.downloads }},
{% endfor %}
];

const userRegionLabels = [
{% for region in download_events_dashboard.user_regions %}
"{{ region.name|escapejs }}",
Expand All @@ -314,6 +347,24 @@ <h4 style="margin:0 0 12px;color:#aaa;text-align:center;">
{% endfor %}
];

const userFileCounts = [
{% for region in download_events_dashboard.user_regions %}
{{ region.file_count }},
{% endfor %}
];

const userZipCounts = [
{% for region in download_events_dashboard.user_regions %}
{{ region.zip_count }},
{% endfor %}
];

const userTotalCounts = [
{% for region in download_events_dashboard.user_regions %}
{{ region.downloads }},
{% endfor %}
];

new Chart(
document.getElementById("gbOverTimeChart"),
{
Expand Down Expand Up @@ -406,6 +457,17 @@ <h4 style="margin:0 0 12px;color:#aaa;text-align:center;">
...commonOptions.plugins,
legend: {
display: false
},
tooltip: {
...commonOptions.plugins.tooltip,
callbacks: {
footer: function (items) {
const i = items[0].dataIndex;
return "Files: " + storageFileCounts[i] +
" · Zips: " + storageZipCounts[i] +
" · Total: " + storageTotalCounts[i];
}
}
}
},
scales: {
Expand Down Expand Up @@ -456,6 +518,20 @@ <h4 style="margin:0 0 12px;color:#aaa;text-align:center;">
options: {
...commonOptions,
indexAxis: "y",
plugins: {
...commonOptions.plugins,
tooltip: {
...commonOptions.plugins.tooltip,
callbacks: {
footer: function (items) {
const i = items[0].dataIndex;
return "Files: " + userFileCounts[i] +
" · Zips: " + userZipCounts[i] +
" · Total: " + userTotalCounts[i];
}
}
}
},
scales: {

x: {
Expand Down
15 changes: 13 additions & 2 deletions osf/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -736,18 +736,27 @@ def _format_bucket_label(self, value, bucket_size):
return value.strftime('%Y-%m-%d')

def _build_region_breakdown(self, queryset, field_name):
"""Grouped in the database — the range can cover millions of rows."""
"""Grouped in the database — the range can cover millions of rows.

`downloads` is the total request count; `file_count` and `zip_count` split it by
request type (a zip is either a folder or a whole-project zip), so file + zip always
equals the total.
"""
rows = queryset.values(field_name).annotate(
downloads=Count('id'),
total_bytes=Sum('size_bytes'),
file_count=Count('id', filter=Q(download_type=DownloadEvent.FILE)),
zip_count=Count('id', filter=~Q(download_type=DownloadEvent.FILE)),
)

breakdown = defaultdict(lambda: {'downloads': 0, 'gb': 0.0})
breakdown = defaultdict(lambda: {'downloads': 0, 'gb': 0.0, 'file_count': 0, 'zip_count': 0})
for row in rows:
# blank and null both mean "we could not tell", so they fold together
region_name = (row[field_name] or 'Unknown').strip() or 'Unknown'
breakdown[region_name]['downloads'] += row['downloads']
breakdown[region_name]['gb'] += (row['total_bytes'] or 0) / (1024**3)
breakdown[region_name]['file_count'] += row['file_count']
breakdown[region_name]['zip_count'] += row['zip_count']

ordered = sorted(breakdown.items(), key=lambda item: item[1]['gb'], reverse=True)[:10]
max_gb = max((data['gb'] for _, data in ordered), default=0)
Expand All @@ -756,6 +765,8 @@ def _build_region_breakdown(self, queryset, field_name):
{
'name': name,
'downloads': data['downloads'],
'file_count': data['file_count'],
'zip_count': data['zip_count'],
'gb': round(data['gb'], 2),
'gb_percent': self._percent(data['gb'], max_gb),
'download_percent': self._percent(data['downloads'], max_downloads),
Expand Down
34 changes: 34 additions & 0 deletions tests/test_download_events_dashboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,40 @@ def test_blank_storage_provider_folds_into_unknown(self):

assert [row['name'] for row in providers] == ['Unknown']

def test_region_breakdown_splits_requests_by_type(self):
# Germany: 2 files + 1 folder zip + 1 project zip = 4 total, 3 zips
make_event(storage_region='Germany', download_type=DownloadEvent.FILE)
make_event(storage_region='Germany', download_type=DownloadEvent.FILE)
make_event(storage_region='Germany', download_type=DownloadEvent.FOLDER_ZIP)
make_event(storage_region='Germany', download_type=DownloadEvent.PROJECT)

regions = self.admin.get_dashboard_data(DownloadEvent.objects.all())['storage_regions']
germany = next(r for r in regions if r['name'] == 'Germany')

assert germany['file_count'] == 2
assert germany['zip_count'] == 2
# file + zip always equals the total — the invariant QA will check
assert germany['file_count'] + germany['zip_count'] == germany['downloads'] == 4

def test_region_type_counts_match_the_period_totals(self):
"""Summed across regions, file/zip counts equal the header's split totals — so the
subtitle and the bars can never disagree."""
make_event(storage_region='Germany', download_type=DownloadEvent.FILE)
make_event(storage_region='United States', download_type=DownloadEvent.FILE)
make_event(storage_region='Germany', download_type=DownloadEvent.FOLDER_ZIP)

data = self.admin.get_dashboard_data(DownloadEvent.objects.all())
regions = data['storage_regions']

assert sum(r['file_count'] for r in regions) == data['split']['file']['count'] == 2
assert sum(r['zip_count'] for r in regions) == data['split']['zip']['count'] == 1

def test_region_type_counts_present_even_when_empty(self):
data = self.admin.get_dashboard_data(DownloadEvent.objects.none())

assert data['storage_regions'] == []
assert data['user_regions'] == []

def test_blank_and_null_regions_fold_into_unknown(self):
make_event(storage_region='')
make_event(storage_region=' ')
Expand Down
Loading