junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-12 13:29:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Stream the Download Package ZIP via /tmp and multipart upload 🟪

Browse source 
Multi-GB packages are no longer held in memory: each source file is read,
written into the on-disk ZIP, and released; the archive is streamed up from
disk. Behaviour unchanged (moto tests green).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Khalim Conn-Kowlessar 2026-07-08 09:51:26 +00:00
parent 879581bb32
commit a36aa49c2b
1 changed files with 15 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
orchestration/bulk_document_download_orchestrator.py
View file

@ -10,7 +10,8 @@ rows are reported, not fatal; a wholly-empty selection is a recorded failure
from __future__ import annotations
import io
import os
import tempfile
import zipfile
from dataclasses import dataclass
from datetime import datetime
@ -99,15 +100,20 @@ class BulkDocumentDownloadOrchestrator:
},
)
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
for entry in plan.entries:
archive.writestr(
entry.zip_path, self._documents.read(entry.s3_bucket, entry.s3_key)
)
# Stream to a /tmp file and multipart-upload it (ADR-0060): a Download
# Package can be several GB, so it is never held whole in memory — each
# source file is read, written into the on-disk ZIP, and released; the
# finished archive is streamed up from disk.
key = f"{self._package_key_prefix}/{package_name}.zip"
self._packages.put_object(key, buffer.getvalue())
with tempfile.TemporaryDirectory() as tmp_dir:
zip_path = os.path.join(tmp_dir, "package.zip")
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as archive:
for entry in plan.entries:
archive.writestr(
entry.zip_path,
self._documents.read(entry.s3_bucket, entry.s3_key),
)
self._packages.upload_file(zip_path, key)
url = self._packages.generate_presigned_url(key, self._url_ttl_seconds)
self._email.send(