j3d1/webvirtcloud
1
0
Fork 0
mirror of https://github.com/retspen/webvirtcloud synced 2026-11-08 19:26:48 +00:00
Code Issues Projects Releases Wiki Activity

perf: the Logs and VM report downloads are written row by row

Browse source 
The workbook kept every cell as an object: a 100 000-row Logs download
took some 400 MB in a web worker. openpyxl's write-only mode writes the
rows as they come (85 MB for the same file, measured), and the Logs rows
are read as tuples, not model objects. The bold, frozen, filtered header,
the column widths and the text cells stay as they were.
This commit is contained in:
catborise 2026-11-05 18:42:17 +03:00
parent d106a4f7ba
commit d15842a250
3 changed files with 41 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

10
admin/tests/test_logs_page.py
View file

@ -79,6 +79,16 @@ class LogsPageTests(TestCase):
self.assertEqual([row[1] for row in values[1:]], ["mallory", "alice"])
# a message starting with = stays text
self.assertEqual(sheet.cell(row=2, column=6).data_type, "s")
# a bold header that stays in view, with filters over every row
self.assertEqual((sheet.freeze_panes, sheet.auto_filter.ref, sheet["A1"].font.b), ("A2", "A1:H3", True))
def test_the_workbook_is_written_row_by_row(self):
# openpyxl's write-only mode: a large download does not keep every cell in memory
from openpyxl import Workbook
with patch("instances.report.Workbook", wraps=Workbook) as book:
self.client.get(reverse("admin:logs_export"))
book.assert_called_once_with(write_only=True)
page = self.page(q="alice", failed="1")
self.assertEqual(page.xpath("//a[contains(@href, 'export')]/@href"),
[reverse("admin:logs_export") + "?q=alice&failed=1"])

9
admin/views.py
View file

@ -306,9 +306,12 @@ def logs_export(request):
rows, text, failed, _query = _logs(request)
zone = timezone.get_current_timezone_name()
header = [f"Date ({zone})", "User", "Host", "VM", "VM UUID", "Message", "Result", "IP"]
logs = list(rows[:EXPORT_ROWS + 1])
data = [[timezone.localtime(log.date).strftime("%Y-%m-%d %H:%M:%S"), log.user, log.host, log.instance,
log.instance_uuid, log.message, str(_result(log.success)), log.ip or ""] for log in logs[:EXPORT_ROWS]]
# tuples, not model objects: up to EXPORT_ROWS of them are held at once
logs = list(rows.values_list("date", "user", "host", "instance", "instance_uuid", "message", "success", "ip")
[:EXPORT_ROWS + 1])
data = [[timezone.localtime(date).strftime("%Y-%m-%d %H:%M:%S"), user, host, instance, uuid, message,
str(_result(success)), ip or ""]
for date, user, host, instance, uuid, message, success, ip in logs[:EXPORT_ROWS]]
notes = [f"Search: {text}" if text else "Search: none", "Failed only" if failed else "All results",
f"Rows: {len(data)}", f"Times in {zone}"]
if len(logs) > EXPORT_ROWS: