perf: the Logs and VM report downloads are written row by row

The workbook kept every cell as an object: a 100 000-row Logs download
took some 400 MB in a web worker. openpyxl's write-only mode writes the
rows as they come (85 MB for the same file, measured), and the Logs rows
are read as tuples, not model objects. The bold, frozen, filtered header,
the column widths and the text cells stay as they were.

admin/tests/test_logs_page.py

@@ -79,6 +79,16 @@ class LogsPageTests(TestCase):
         self.assertEqual([row[1] for row in values[1:]], ["mallory", "alice"])
         # a message starting with = stays text
         self.assertEqual(sheet.cell(row=2, column=6).data_type, "s")
+        # a bold header that stays in view, with filters over every row
+        self.assertEqual((sheet.freeze_panes, sheet.auto_filter.ref, sheet["A1"].font.b), ("A2", "A1:H3", True))
+
+    def test_the_workbook_is_written_row_by_row(self):
+        # openpyxl's write-only mode: a large download does not keep every cell in memory
+        from openpyxl import Workbook
+
+        with patch("instances.report.Workbook", wraps=Workbook) as book:
+            self.client.get(reverse("admin:logs_export"))
+        book.assert_called_once_with(write_only=True)
         page = self.page(q="alice", failed="1")
         self.assertEqual(page.xpath("//a[contains(@href, 'export')]/@href"),
                          [reverse("admin:logs_export") + "?q=alice&failed=1"])

admin/views.py

@@ -306,9 +306,12 @@ def logs_export(request):
     rows, text, failed, _query = _logs(request)
     zone = timezone.get_current_timezone_name()
     header = [f"Date ({zone})", "User", "Host", "VM", "VM UUID", "Message", "Result", "IP"]
-    logs = list(rows[:EXPORT_ROWS + 1])
+    # tuples, not model objects: up to EXPORT_ROWS of them are held at once
-    data = [[timezone.localtime(log.date).strftime("%Y-%m-%d %H:%M:%S"), log.user, log.host, log.instance,
+    logs = list(rows.values_list("date", "user", "host", "instance", "instance_uuid", "message", "success", "ip")
-             log.instance_uuid, log.message, str(_result(log.success)), log.ip or ""] for log in logs[:EXPORT_ROWS]]
+                [:EXPORT_ROWS + 1])
+    data = [[timezone.localtime(date).strftime("%Y-%m-%d %H:%M:%S"), user, host, instance, uuid, message,
+             str(_result(success)), ip or ""]
+            for date, user, host, instance, uuid, message, success, ip in logs[:EXPORT_ROWS]]
     notes = [f"Search: {text}" if text else "Search: none", "Failed only" if failed else "All results",
              f"Rows: {len(data)}", f"Times in {zone}"]
     if len(logs) > EXPORT_ROWS:

instances/report.py

@@ -3,6 +3,7 @@
 from io import BytesIO
 
 from openpyxl import Workbook
+from openpyxl.cell import WriteOnlyCell
 from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
 from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
@@ -11,31 +12,34 @@ MAX_WIDTH = 60
 
 
 def workbook(title, header, rows, notes_title, notes):
-    """The .xlsx file's bytes. Every text cell stays text: openpyxl would
+    """The .xlsx file's bytes, written row by row (openpyxl's write-only mode:
-    write one starting with "=" as a formula, and the text comes from VM
+    a large download does not hold every cell in memory). Every text cell
-    names and guests."""
+    stays text: openpyxl would write one starting with "=" as a formula, and
-    book = Workbook()
+    the text comes from VM names and guests."""
-    sheet = book.active
+    book = Workbook(write_only=True)
-    sheet.title = title
+    sheet = book.create_sheet(title)
-    for values in [header, *rows]:
+    # control characters (a name typed at a failed login) cannot be in the file: written as \x01
-        # a control character (a name typed at a failed login) cannot be in the file: written as \x01
+    rows = [[ILLEGAL_CHARACTERS_RE.sub(lambda m: f"\\x{ord(m.group()):02x}", value) if isinstance(value, str) else value
-        sheet.append([ILLEGAL_CHARACTERS_RE.sub(lambda m: f"\\x{ord(m.group()):02x}", value)
+             for value in values] for values in [header, *rows]]
-                      if isinstance(value, str) else value for value in values])
+    for index, column in enumerate(zip(*rows), start=1):
-    for row in sheet.iter_rows():
+        width = max(len(str(value or "")) for value in column) + 2
-        for cell in row:
-            if isinstance(cell.value, str):
-                cell.data_type = "s"
-    for cell in sheet[1]:
-        cell.font = Font(bold=True)
-    sheet.freeze_panes = "A2"
-    sheet.auto_filter.ref = sheet.dimensions
-    for index, column in enumerate(sheet.columns, start=1):
-        width = max(len(str(cell.value or "")) for cell in column) + 2
         sheet.column_dimensions[get_column_letter(index)].width = min(width, MAX_WIDTH)
+    sheet.freeze_panes = "A2"
+    for number, values in enumerate(rows):
+        cells = []
+        for value in values:
+            cell = WriteOnlyCell(sheet, value=value)
+            if isinstance(value, str):
+                cell.data_type = "s"
+            if number == 0:
+                cell.font = Font(bold=True)
+            cells.append(cell)
+        sheet.append(cells)
+    sheet.auto_filter.ref = f"A1:{get_column_letter(len(header))}{len(rows)}"
     notes_sheet = book.create_sheet(notes_title)
+    notes_sheet.column_dimensions["A"].width = MAX_WIDTH * 2
     for line in notes:
         notes_sheet.append([line])
-    notes_sheet.column_dimensions["A"].width = MAX_WIDTH * 2
     out = BytesIO()
     book.save(out)
     return out.getvalue()