mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
91db972d48
First successful live run surfaced three issues: 1. Email looked rubbish (a giant raw presigned URL). Now sends a proper HTML email with a 'Download documents' button plus a plain-text fallback, and a summary (N documents across M properties, expiry). Email delivery is now best-effort: a transport failure no longer loses an already-built package (the link is still on sub_task.outputs), and the SMTP connect has a 30s timeout so an unreachable SES endpoint fails fast instead of hanging to the 900s Lambda timeout. 2. Every folder was 'address unavailable (...)': the resolver read property.address, but these are HubSpot deals with no property row. It now uses the deal's dealname from hubspot_deal_data. 3. No logs / no idea why a run took ~9 minutes: the worker's INFO logs were dropped (Lambda root logger defaults to WARNING). The handler now raises the level, and the orchestrator logs per-phase timing and volume (gather+plan, packaged N files / X MB, upload, email, total) so the slow phase is visible. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
276 lines
11 KiB
Python
276 lines
11 KiB
Python
"""The Bulk Document Download orchestrator (ADR-0060).
|
|
|
|
Ties the pieces together for one run: gather a property set's uploaded files
|
|
(by `landlord_property_id`), enrich each with its display address, lay them out
|
|
with the Download Package plan, stream the chosen files into a single ZIP,
|
|
upload it, mint a presigned URL, and email the requester. Best-effort — skipped
|
|
rows are reported, not fatal; a wholly-empty selection is a recorded failure
|
|
(ADR-0060).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
from dataclasses import dataclass
|
|
from typing import Protocol, Sequence
|
|
|
|
from domain.bulk_document_download.package_plan import (
|
|
ResolvedDocument,
|
|
SkippedDocument,
|
|
plan_download_package,
|
|
)
|
|
from domain.bulk_document_download.property_document import PropertyDocument
|
|
from domain.tasks.subtasks import SubTaskFailure
|
|
from infrastructure.s3.s3_client import S3Client
|
|
from repositories.email.email_sender import EmailSender
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class UploadedFileReader(Protocol):
|
|
def by_landlord_property_ids(
|
|
self, landlord_property_ids: Sequence[str]
|
|
) -> list[PropertyDocument]: ...
|
|
|
|
|
|
class AddressResolver(Protocol):
|
|
def address_for(self, landlord_property_id: str) -> str: ...
|
|
|
|
|
|
class DocumentDownloader(Protocol):
|
|
def download(self, bucket: str, key: str, dest_path: str) -> None: ...
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DownloadPackageResult:
|
|
"""What a completed run reports (lands in `sub_task.outputs`, ADR-0060)."""
|
|
|
|
presigned_url: str
|
|
package_s3_key: str
|
|
included: int
|
|
skipped: tuple[SkippedDocument, ...]
|
|
|
|
|
|
class BulkDocumentDownloadOrchestrator:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
uploaded_files: UploadedFileReader,
|
|
addresses: AddressResolver,
|
|
documents: DocumentDownloader,
|
|
packages: S3Client,
|
|
email: EmailSender,
|
|
url_ttl_seconds: int,
|
|
package_key_prefix: str = "bulk-downloads",
|
|
) -> None:
|
|
self._uploaded_files = uploaded_files
|
|
self._addresses = addresses
|
|
self._documents = documents
|
|
self._packages = packages
|
|
self._email = email
|
|
self._url_ttl_seconds = url_ttl_seconds
|
|
self._package_key_prefix = package_key_prefix
|
|
|
|
def run(
|
|
self,
|
|
*,
|
|
landlord_property_ids: Sequence[str],
|
|
recipient_email: str,
|
|
package_name: str,
|
|
) -> DownloadPackageResult:
|
|
started = time.monotonic()
|
|
documents = self._uploaded_files.by_landlord_property_ids(
|
|
landlord_property_ids
|
|
)
|
|
plan = plan_download_package([self._resolve(doc) for doc in documents])
|
|
|
|
skipped: list[SkippedDocument] = list(plan.skipped)
|
|
null_type_skips = len(skipped)
|
|
|
|
logger.info(
|
|
"bulk_document_download: selected=%d matched_documents=%d "
|
|
"planned_entries=%d null_type_skips=%d (gather+plan %.1fs)",
|
|
len(landlord_property_ids),
|
|
len(documents),
|
|
len(plan.entries),
|
|
null_type_skips,
|
|
time.monotonic() - started,
|
|
)
|
|
|
|
if not plan.entries:
|
|
raise self._empty_failure(
|
|
selected=len(landlord_property_ids),
|
|
matched=len(documents),
|
|
skipped=skipped,
|
|
)
|
|
|
|
# Stream to a /tmp file and multipart-upload it (ADR-0060): a Download
|
|
# Package can be several GB, so it is never held whole in memory. Each
|
|
# source file is streamed to its own temp file, added to the on-disk ZIP,
|
|
# then deleted; the finished archive is streamed up from disk. Best-effort
|
|
# (ADR-0060): a file that can't be read — deleted, or in a bucket the role
|
|
# can't reach — becomes a SkippedDocument, it does not fail the package.
|
|
key = f"{self._package_key_prefix}/{package_name}.zip"
|
|
included = 0
|
|
zip_bytes = 0
|
|
zip_started = time.monotonic()
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
zip_path = os.path.join(tmp_dir, "package.zip")
|
|
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
for index, entry in enumerate(plan.entries):
|
|
member_path = os.path.join(tmp_dir, f"member-{index}")
|
|
try:
|
|
self._documents.download(
|
|
entry.s3_bucket, entry.s3_key, member_path
|
|
)
|
|
except Exception:
|
|
skipped.append(
|
|
SkippedDocument(
|
|
landlord_property_id=entry.landlord_property_id,
|
|
s3_key=entry.s3_key,
|
|
reason="unreadable",
|
|
)
|
|
)
|
|
continue
|
|
archive.write(member_path, arcname=entry.zip_path)
|
|
os.remove(member_path)
|
|
included += 1
|
|
|
|
if included == 0:
|
|
# Every file that resolved failed to read — no package to send.
|
|
raise self._empty_failure(
|
|
selected=len(landlord_property_ids),
|
|
matched=len(documents),
|
|
skipped=skipped,
|
|
)
|
|
zip_bytes = os.path.getsize(zip_path)
|
|
logger.info(
|
|
"bulk_document_download: packaged %d files (%d unreadable), "
|
|
"%.1f MB in %.1fs",
|
|
included,
|
|
len(skipped) - null_type_skips,
|
|
zip_bytes / 1_000_000,
|
|
time.monotonic() - zip_started,
|
|
)
|
|
upload_started = time.monotonic()
|
|
self._packages.upload_file(zip_path, key)
|
|
logger.info(
|
|
"bulk_document_download: uploaded %.1f MB in %.1fs",
|
|
zip_bytes / 1_000_000,
|
|
time.monotonic() - upload_started,
|
|
)
|
|
url = self._packages.generate_presigned_url(key, self._url_ttl_seconds)
|
|
|
|
property_count = len({entry.landlord_property_id for entry in plan.entries})
|
|
subject, body, html_body = self._compose_email(
|
|
url=url,
|
|
included=included,
|
|
properties=property_count,
|
|
skipped=len(skipped),
|
|
)
|
|
try:
|
|
self._email.send(
|
|
to=recipient_email,
|
|
subject=subject,
|
|
body=body,
|
|
html_body=html_body,
|
|
)
|
|
logger.info("bulk_document_download: emailed %s", recipient_email)
|
|
except Exception:
|
|
# Best-effort delivery (ADR-0060): the link is also written to
|
|
# sub_task.outputs, so an email/transport failure must not lose a
|
|
# package that was already built and uploaded.
|
|
logger.warning(
|
|
"bulk_document_download: email delivery to %s failed; the link is "
|
|
"still recorded on sub_task.outputs",
|
|
recipient_email,
|
|
exc_info=True,
|
|
)
|
|
|
|
logger.info(
|
|
"bulk_document_download: done — %d documents, %.1f MB, total %.1fs",
|
|
included,
|
|
zip_bytes / 1_000_000,
|
|
time.monotonic() - started,
|
|
)
|
|
return DownloadPackageResult(
|
|
presigned_url=url,
|
|
package_s3_key=key,
|
|
included=included,
|
|
skipped=tuple(skipped),
|
|
)
|
|
|
|
def _compose_email(
|
|
self, *, url: str, included: int, properties: int, skipped: int
|
|
) -> tuple[str, str, str]:
|
|
"""The delivery email (ADR-0059): a subject, a plain-text body, and an
|
|
HTML alternative with a clean download button so the long presigned URL
|
|
isn't the visible content."""
|
|
minutes = self._url_ttl_seconds // 60
|
|
noun = "property" if properties == 1 else "properties"
|
|
skip_note = f" {skipped} item(s) were skipped." if skipped else ""
|
|
subject = f"Your document download is ready ({included} documents)"
|
|
body = (
|
|
"Your document package is ready.\n\n"
|
|
f"It contains {included} document(s) across {properties} {noun}."
|
|
f"{skip_note}\n\n"
|
|
f"Download it here (link valid for {minutes} minutes):\n{url}\n"
|
|
)
|
|
safe_url = html.escape(url, quote=True)
|
|
html_body = (
|
|
'<div style="font-family:Arial,Helvetica,sans-serif;font-size:15px;'
|
|
'color:#1a1a1a;line-height:1.5;">'
|
|
"<p>Your document package is ready.</p>"
|
|
f"<p>It contains <strong>{included} document(s)</strong> across "
|
|
f"<strong>{properties}</strong> {noun}.{skip_note}</p>"
|
|
'<p style="margin:24px 0;">'
|
|
f'<a href="{safe_url}" style="display:inline-block;padding:12px 22px;'
|
|
"background:#0b8457;color:#ffffff;text-decoration:none;border-radius:6px;"
|
|
'font-weight:bold;">Download documents</a></p>'
|
|
f'<p style="color:#666;font-size:13px;">This link expires in {minutes} '
|
|
"minutes. If the button doesn't work, paste this URL into your browser:"
|
|
f'<br><span style="word-break:break-all;">{safe_url}</span></p>'
|
|
"</div>"
|
|
)
|
|
return subject, body, html_body
|
|
|
|
def _empty_failure(
|
|
self, *, selected: int, matched: int, skipped: list[SkippedDocument]
|
|
) -> SubTaskFailure:
|
|
"""A recorded failure when the selection yields no readable documents
|
|
(ADR-0060) — never an empty ZIP emailed to the user. The message carries
|
|
the stage counts (so they surface in the worker's WARNING log) and the
|
|
skip detail rides on the failure's ``details`` (→ ``sub_task.outputs``)."""
|
|
return SubTaskFailure(
|
|
f"no documents could be packaged for the selection "
|
|
f"(selected {selected} properties, matched {matched} documents, "
|
|
f"{len(skipped)} skipped)",
|
|
details={
|
|
"selected_properties": selected,
|
|
"matched_documents": matched,
|
|
"planned_entries": 0,
|
|
"skipped": [
|
|
{
|
|
"landlord_property_id": s.landlord_property_id,
|
|
"s3_key": s.s3_key,
|
|
"reason": s.reason,
|
|
}
|
|
for s in skipped
|
|
],
|
|
},
|
|
)
|
|
|
|
def _resolve(self, document: PropertyDocument) -> ResolvedDocument:
|
|
return ResolvedDocument(
|
|
landlord_property_id=document.landlord_property_id,
|
|
address=self._addresses.address_for(document.landlord_property_id),
|
|
document_type=document.document_type,
|
|
s3_bucket=document.s3_bucket,
|
|
s3_key=document.s3_key,
|
|
uploaded_at=document.uploaded_at,
|
|
)
|