mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
9ec7987e97
Addresses PR #1498 review: - A per-file read failure (missing/deleted object, or a bucket the role can't reach) is now a SkippedDocument(reason=unreadable), not a whole-run abort; an all-unreadable selection still fails (no empty package). Best-effort on the read path (ADR-0060). - Each ZIP member is streamed to a temp file (S3DocumentDownloader.download, boto download_file) and added from disk, so a single multi-GB member never hits the heap — the 'never held whole in memory' claim is now true. - The 409 double-submit guard is DB-arbitrated: the sub_task insert is conditional on the task having none, so a race creates only one. - Local env gets a placeholder recipient email so the route is exercisable. - PackageEntry carries landlord_property_id so a read failure can be reported. - TODO noting the UploadedFile.s3_upload_timestamp typing fix. Two new tests: per-file read failure skips-and-reports; address fallback flows to the folder name. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
100 lines
4 KiB
Python
100 lines
4 KiB
Python
"""bulk_document_download Lambda (ADR-0060).
|
|
|
|
Attach-mode (ADR-0055) handler. The FastAPI route created the app-owned Task +
|
|
SubTask and pinned the recipe onto the SubTask's ``inputs``; this reads that
|
|
recipe, assembles the Download Package through the orchestrator, and returns the
|
|
result (presigned URL + skipped-document summary), which `TaskOrchestrator`
|
|
records on ``sub_task.outputs``. A wholly-empty selection raises
|
|
``SubTaskFailure`` from the orchestrator — recorded, not retried.
|
|
|
|
PostgresConfig-only (POSTGRES_*), like the bulk_upload_finaliser Lambda. Source
|
|
documents are read from each row's own ``s3_file_bucket``; the ZIP is written to
|
|
the dedicated ``DOCUMENT_EXPORTS_BUCKET``. SES SMTP settings arrive as env vars
|
|
(terraform bakes the
|
|
Secrets Manager IAM-user credentials in).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
import boto3
|
|
|
|
from applications.bulk_document_download.bulk_document_download_trigger_body import (
|
|
BulkDocumentDownloadTriggerBody,
|
|
)
|
|
from domain.tasks.tasks import Source
|
|
from infrastructure.email.ses_smtp_email_sender import SesSmtpEmailSender
|
|
from infrastructure.postgres.config import PostgresConfig
|
|
from infrastructure.postgres.engine import make_engine, make_session
|
|
from infrastructure.s3.s3_client import S3Client
|
|
from infrastructure.s3.s3_document_downloader import S3DocumentDownloader
|
|
from orchestration.bulk_document_download_orchestrator import (
|
|
BulkDocumentDownloadOrchestrator,
|
|
)
|
|
from repositories.property.landlord_address_resolver import LandlordAddressResolver
|
|
from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository
|
|
from repositories.uploaded_file.uploaded_file_postgres_repository import (
|
|
UploadedFilePostgresRepository,
|
|
)
|
|
from utilities.aws_lambda.task_handler import task_handler
|
|
|
|
# The Download Package link is valid for 60 minutes (ADR-0060).
|
|
_URL_TTL_SECONDS = 3600
|
|
|
|
|
|
def _email_sender() -> SesSmtpEmailSender:
|
|
return SesSmtpEmailSender(
|
|
host=os.environ["SES_SMTP_HOST"],
|
|
port=int(os.environ["SES_SMTP_PORT"]),
|
|
username=os.environ["SES_SMTP_USERNAME"],
|
|
password=os.environ["SES_SMTP_PASSWORD"],
|
|
from_address=os.environ["SES_SMTP_FROM_ADDRESS"],
|
|
)
|
|
|
|
|
|
@task_handler(task_source="bulk_document_download", source=Source.PORTFOLIO)
|
|
def handler(body: dict[str, Any], context: Any) -> dict[str, Any]:
|
|
trigger = BulkDocumentDownloadTriggerBody.model_validate(body)
|
|
engine = make_engine(PostgresConfig.from_env(os.environ))
|
|
session = make_session(engine)
|
|
try:
|
|
subtask = SubTaskPostgresRepository(session).get(trigger.subtask_id)
|
|
recipe: dict[str, Any] = subtask.inputs or {}
|
|
landlord_property_ids: list[str] = [
|
|
str(x) for x in recipe["landlord_property_ids"]
|
|
]
|
|
recipient_email: str = str(recipe["recipient_email"])
|
|
package_name: str = str(recipe["package_name"])
|
|
|
|
boto_s3: Any = boto3.client("s3") # pyright: ignore[reportUnknownMemberType]
|
|
orchestrator = BulkDocumentDownloadOrchestrator(
|
|
uploaded_files=UploadedFilePostgresRepository(session),
|
|
addresses=LandlordAddressResolver(session, landlord_property_ids),
|
|
documents=S3DocumentDownloader(boto_s3),
|
|
packages=S3Client(boto_s3, os.environ["DOCUMENT_EXPORTS_BUCKET"]),
|
|
email=_email_sender(),
|
|
url_ttl_seconds=_URL_TTL_SECONDS,
|
|
)
|
|
result = orchestrator.run(
|
|
landlord_property_ids=landlord_property_ids,
|
|
recipient_email=recipient_email,
|
|
package_name=package_name,
|
|
)
|
|
finally:
|
|
session.close()
|
|
|
|
return {
|
|
"presigned_url": result.presigned_url,
|
|
"package_s3_key": result.package_s3_key,
|
|
"included": result.included,
|
|
"skipped": [
|
|
{
|
|
"landlord_property_id": skip.landlord_property_id,
|
|
"s3_key": skip.s3_key,
|
|
"reason": skip.reason,
|
|
}
|
|
for skip in result.skipped
|
|
],
|
|
}
|