diff --git a/admin/tests/test_logs_page.py b/admin/tests/test_logs_page.py index 26c8155d..d371da17 100644 --- a/admin/tests/test_logs_page.py +++ b/admin/tests/test_logs_page.py @@ -79,6 +79,16 @@ class LogsPageTests(TestCase): self.assertEqual([row[1] for row in values[1:]], ["mallory", "alice"]) # a message starting with = stays text self.assertEqual(sheet.cell(row=2, column=6).data_type, "s") + # a bold header that stays in view, with filters over every row + self.assertEqual((sheet.freeze_panes, sheet.auto_filter.ref, sheet["A1"].font.b), ("A2", "A1:H3", True)) + + def test_the_workbook_is_written_row_by_row(self): + # openpyxl's write-only mode: a large download does not keep every cell in memory + from openpyxl import Workbook + + with patch("instances.report.Workbook", wraps=Workbook) as book: + self.client.get(reverse("admin:logs_export")) + book.assert_called_once_with(write_only=True) page = self.page(q="alice", failed="1") self.assertEqual(page.xpath("//a[contains(@href, 'export')]/@href"), [reverse("admin:logs_export") + "?q=alice&failed=1"]) diff --git a/admin/views.py b/admin/views.py index a9bb11cb..2b8937e5 100644 --- a/admin/views.py +++ b/admin/views.py @@ -306,9 +306,12 @@ def logs_export(request): rows, text, failed, _query = _logs(request) zone = timezone.get_current_timezone_name() header = [f"Date ({zone})", "User", "Host", "VM", "VM UUID", "Message", "Result", "IP"] - logs = list(rows[:EXPORT_ROWS + 1]) - data = [[timezone.localtime(log.date).strftime("%Y-%m-%d %H:%M:%S"), log.user, log.host, log.instance, - log.instance_uuid, log.message, str(_result(log.success)), log.ip or ""] for log in logs[:EXPORT_ROWS]] + # tuples, not model objects: up to EXPORT_ROWS of them are held at once + logs = list(rows.values_list("date", "user", "host", "instance", "instance_uuid", "message", "success", "ip") + [:EXPORT_ROWS + 1]) + data = [[timezone.localtime(date).strftime("%Y-%m-%d %H:%M:%S"), user, host, instance, uuid, message, + str(_result(success)), ip or ""] + for date, user, host, instance, uuid, message, success, ip in logs[:EXPORT_ROWS]] notes = [f"Search: {text}" if text else "Search: none", "Failed only" if failed else "All results", f"Rows: {len(data)}", f"Times in {zone}"] if len(logs) > EXPORT_ROWS: diff --git a/instances/report.py b/instances/report.py index 7a5e12b4..749864c8 100644 --- a/instances/report.py +++ b/instances/report.py @@ -3,6 +3,7 @@ from io import BytesIO from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE from openpyxl.styles import Font from openpyxl.utils import get_column_letter @@ -11,31 +12,34 @@ MAX_WIDTH = 60 def workbook(title, header, rows, notes_title, notes): - """The .xlsx file's bytes. Every text cell stays text: openpyxl would - write one starting with "=" as a formula, and the text comes from VM - names and guests.""" - book = Workbook() - sheet = book.active - sheet.title = title - for values in [header, *rows]: - # a control character (a name typed at a failed login) cannot be in the file: written as \x01 - sheet.append([ILLEGAL_CHARACTERS_RE.sub(lambda m: f"\\x{ord(m.group()):02x}", value) - if isinstance(value, str) else value for value in values]) - for row in sheet.iter_rows(): - for cell in row: - if isinstance(cell.value, str): - cell.data_type = "s" - for cell in sheet[1]: - cell.font = Font(bold=True) - sheet.freeze_panes = "A2" - sheet.auto_filter.ref = sheet.dimensions - for index, column in enumerate(sheet.columns, start=1): - width = max(len(str(cell.value or "")) for cell in column) + 2 + """The .xlsx file's bytes, written row by row (openpyxl's write-only mode: + a large download does not hold every cell in memory). Every text cell + stays text: openpyxl would write one starting with "=" as a formula, and + the text comes from VM names and guests.""" + book = Workbook(write_only=True) + sheet = book.create_sheet(title) + # control characters (a name typed at a failed login) cannot be in the file: written as \x01 + rows = [[ILLEGAL_CHARACTERS_RE.sub(lambda m: f"\\x{ord(m.group()):02x}", value) if isinstance(value, str) else value + for value in values] for values in [header, *rows]] + for index, column in enumerate(zip(*rows), start=1): + width = max(len(str(value or "")) for value in column) + 2 sheet.column_dimensions[get_column_letter(index)].width = min(width, MAX_WIDTH) + sheet.freeze_panes = "A2" + for number, values in enumerate(rows): + cells = [] + for value in values: + cell = WriteOnlyCell(sheet, value=value) + if isinstance(value, str): + cell.data_type = "s" + if number == 0: + cell.font = Font(bold=True) + cells.append(cell) + sheet.append(cells) + sheet.auto_filter.ref = f"A1:{get_column_letter(len(header))}{len(rows)}" notes_sheet = book.create_sheet(notes_title) + notes_sheet.column_dimensions["A"].width = MAX_WIDTH * 2 for line in notes: notes_sheet.append([line]) - notes_sheet.column_dimensions["A"].width = MAX_WIDTH * 2 out = BytesIO() book.save(out) return out.getvalue()