diff --git a/orchestration/bulk_document_download_orchestrator.py b/orchestration/bulk_document_download_orchestrator.py index 39ee7271e..ba979a1c6 100644 --- a/orchestration/bulk_document_download_orchestrator.py +++ b/orchestration/bulk_document_download_orchestrator.py @@ -10,7 +10,8 @@ rows are reported, not fatal; a wholly-empty selection is a recorded failure from __future__ import annotations -import io +import os +import tempfile import zipfile from dataclasses import dataclass from datetime import datetime @@ -99,15 +100,20 @@ class BulkDocumentDownloadOrchestrator: }, ) - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: - for entry in plan.entries: - archive.writestr( - entry.zip_path, self._documents.read(entry.s3_bucket, entry.s3_key) - ) - + # Stream to a /tmp file and multipart-upload it (ADR-0060): a Download + # Package can be several GB, so it is never held whole in memory — each + # source file is read, written into the on-disk ZIP, and released; the + # finished archive is streamed up from disk. key = f"{self._package_key_prefix}/{package_name}.zip" - self._packages.put_object(key, buffer.getvalue()) + with tempfile.TemporaryDirectory() as tmp_dir: + zip_path = os.path.join(tmp_dir, "package.zip") + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as archive: + for entry in plan.entries: + archive.writestr( + entry.zip_path, + self._documents.read(entry.s3_bucket, entry.s3_key), + ) + self._packages.upload_file(zip_path, key) url = self._packages.generate_presigned_url(key, self._url_ttl_seconds) self._email.send(