mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
9ec7987e97
Addresses PR #1498 review: - A per-file read failure (missing/deleted object, or a bucket the role can't reach) is now a SkippedDocument(reason=unreadable), not a whole-run abort; an all-unreadable selection still fails (no empty package). Best-effort on the read path (ADR-0060). - Each ZIP member is streamed to a temp file (S3DocumentDownloader.download, boto download_file) and added from disk, so a single multi-GB member never hits the heap — the 'never held whole in memory' claim is now true. - The 409 double-submit guard is DB-arbitrated: the sub_task insert is conditional on the task having none, so a race creates only one. - Local env gets a placeholder recipient email so the route is exercisable. - PackageEntry carries landlord_property_id so a read failure can be reported. - TODO noting the UploadedFile.s3_upload_timestamp typing fix. Two new tests: per-file read failure skips-and-reports; address fallback flows to the folder name. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
104 lines
3.5 KiB
Python
104 lines
3.5 KiB
Python
"""The Download Package plan (ADR-0060).
|
|
|
|
Pure logic: given the documents resolved for a set of properties, decide what
|
|
goes where in the ZIP — one folder per property, the latest file of each
|
|
Document Type — and which rows are skipped and reported. No I/O: the
|
|
orchestrator does the S3/DB work; this module only lays out the archive.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import posixpath
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from typing import Optional, Sequence
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResolvedDocument:
|
|
"""One `uploaded_files` row resolved for packaging: its property identity
|
|
(matched by `landlord_property_id`, ADR-0060), the display address enriched
|
|
from the hubspot deals data, its Document Type (`file_type`, nullable), the
|
|
S3 location to read it from, and when it was uploaded (for latest-per-type).
|
|
"""
|
|
|
|
landlord_property_id: str
|
|
address: str
|
|
document_type: Optional[str]
|
|
s3_bucket: str
|
|
s3_key: str
|
|
uploaded_at: datetime
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PackageEntry:
|
|
"""One file to place in the Download Package: where it lands in the ZIP and
|
|
the S3 object to stream in from. Carries the property identity so a
|
|
read-time failure can be reported as a SkippedDocument (ADR-0060)."""
|
|
|
|
zip_path: str
|
|
s3_bucket: str
|
|
s3_key: str
|
|
landlord_property_id: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SkippedDocument:
|
|
"""A resolved row left out of the package, with the reason (surfaced in the
|
|
run's `sub_task.outputs`, ADR-0060)."""
|
|
|
|
landlord_property_id: str
|
|
s3_key: str
|
|
reason: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DownloadPackagePlan:
|
|
"""The laid-out archive: the files to include and the rows skipped."""
|
|
|
|
entries: tuple[PackageEntry, ...]
|
|
skipped: tuple[SkippedDocument, ...]
|
|
|
|
|
|
def _extension_of(s3_key: str) -> str:
|
|
"""The file extension of an S3 key (``.pdf``, ``.zip``, …), empty when the
|
|
key's basename has none. Used to keep the packaged file openable while
|
|
naming it by Document Type."""
|
|
_root, ext = posixpath.splitext(posixpath.basename(s3_key))
|
|
return ext
|
|
|
|
|
|
def plan_download_package(
|
|
documents: Sequence[ResolvedDocument],
|
|
) -> DownloadPackagePlan:
|
|
"""Lay out a Download Package from resolved documents (ADR-0060): one folder
|
|
per property, the latest file of each Document Type, null-Document-Type rows
|
|
skipped and reported."""
|
|
latest_by_group: dict[tuple[str, str], ResolvedDocument] = {}
|
|
skipped: list[SkippedDocument] = []
|
|
for document in documents:
|
|
if document.document_type is None:
|
|
skipped.append(
|
|
SkippedDocument(
|
|
landlord_property_id=document.landlord_property_id,
|
|
s3_key=document.s3_key,
|
|
reason="null_document_type",
|
|
)
|
|
)
|
|
continue
|
|
group = (document.landlord_property_id, document.document_type)
|
|
incumbent = latest_by_group.get(group)
|
|
if incumbent is None or document.uploaded_at > incumbent.uploaded_at:
|
|
latest_by_group[group] = document
|
|
|
|
entries: list[PackageEntry] = [
|
|
PackageEntry(
|
|
zip_path=f"{document.address} ({document.landlord_property_id})/"
|
|
f"{document.document_type}{_extension_of(document.s3_key)}",
|
|
s3_bucket=document.s3_bucket,
|
|
s3_key=document.s3_key,
|
|
landlord_property_id=document.landlord_property_id,
|
|
)
|
|
for document in latest_by_group.values()
|
|
]
|
|
return DownloadPackagePlan(entries=tuple(entries), skipped=tuple(skipped))
|