diff --git a/applications/bulk_document_download/Dockerfile b/applications/bulk_document_download/Dockerfile new file mode 100644 index 000000000..e65d1bd9a --- /dev/null +++ b/applications/bulk_document_download/Dockerfile @@ -0,0 +1,35 @@ +FROM public.ecr.aws/lambda/python:3.11 + +# Postgres host/port/database are baked into the image at build time from the +# deploy workflow's --build-arg values (GitHub Actions DEV_DB_* secrets), +# mirroring the bulk_upload_finaliser Dockerfile. They map onto the POSTGRES_* +# names PostgresConfig.from_env reads. Username/password are NOT baked in -- +# Terraform injects those (and the SES SMTP credentials) as Lambda env vars from +# Secrets Manager. +ARG DEV_DB_HOST +ARG DEV_DB_PORT +ARG DEV_DB_NAME + +ENV POSTGRES_HOST=${DEV_DB_HOST} +ENV POSTGRES_PORT=${DEV_DB_PORT} +ENV POSTGRES_DATABASE=${DEV_DB_NAME} + +WORKDIR /var/task + +COPY applications/bulk_document_download/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# DDD-shaped packages only -- no pandas, no legacy backend/. +COPY datatypes/ datatypes/ +COPY domain/ domain/ +COPY infrastructure/ infrastructure/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY utilities/ utilities/ +COPY applications/ applications/ + +# Place the handler at the Lambda task root so the runtime resolves +# ``main.handler`` without an extra package prefix. +COPY applications/bulk_document_download/handler.py /var/task/main.py + +CMD ["main.handler"] diff --git a/applications/bulk_document_download/bulk_document_download_trigger_body.py b/applications/bulk_document_download/bulk_document_download_trigger_body.py new file mode 100644 index 000000000..95da349b5 --- /dev/null +++ b/applications/bulk_document_download/bulk_document_download_trigger_body.py @@ -0,0 +1,19 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class BulkDocumentDownloadTriggerBody(BaseModel): + """SQS trigger for the bulk_document_download Lambda (ADR-0055/0060). + + Attach mode: the FastAPI route created the Task + SubTask and pinned the + recipe (``landlord_property_ids``, ``recipient_email``, ``package_name``) + onto the SubTask's ``inputs``. The message therefore carries only the + identifiers — the (potentially large) property set never travels through the + 256 KB SQS body. + """ + + model_config = ConfigDict(extra="allow") + + task_id: UUID + subtask_id: UUID diff --git a/applications/bulk_document_download/handler.py b/applications/bulk_document_download/handler.py new file mode 100644 index 000000000..c5af7014c --- /dev/null +++ b/applications/bulk_document_download/handler.py @@ -0,0 +1,100 @@ +"""bulk_document_download Lambda (ADR-0060). + +Attach-mode (ADR-0055) handler. The FastAPI route created the app-owned Task + +SubTask and pinned the recipe onto the SubTask's ``inputs``; this reads that +recipe, assembles the Download Package through the orchestrator, and returns the +result (presigned URL + skipped-document summary), which `TaskOrchestrator` +records on ``sub_task.outputs``. A wholly-empty selection raises +``SubTaskFailure`` from the orchestrator — recorded, not retried. + +PostgresConfig-only (POSTGRES_*), like the bulk_upload_finaliser Lambda. Source +documents are read from each row's own ``s3_file_bucket``; the ZIP is written to +the dedicated ``DOCUMENT_EXPORTS_BUCKET``. SES SMTP settings arrive as env vars +(terraform bakes the +Secrets Manager IAM-user credentials in). +""" + +from __future__ import annotations + +import os +from typing import Any + +import boto3 + +from applications.bulk_document_download.bulk_document_download_trigger_body import ( + BulkDocumentDownloadTriggerBody, +) +from domain.tasks.tasks import Source +from infrastructure.email.ses_smtp_email_sender import SesSmtpEmailSender +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine, make_session +from infrastructure.s3.s3_client import S3Client +from infrastructure.s3.s3_document_bytes_reader import S3DocumentBytesReader +from orchestration.bulk_document_download_orchestrator import ( + BulkDocumentDownloadOrchestrator, +) +from repositories.property.landlord_address_resolver import LandlordAddressResolver +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.uploaded_file.uploaded_file_postgres_repository import ( + UploadedFilePostgresRepository, +) +from utilities.aws_lambda.task_handler import task_handler + +# The Download Package link is valid for 60 minutes (ADR-0060). +_URL_TTL_SECONDS = 3600 + + +def _email_sender() -> SesSmtpEmailSender: + return SesSmtpEmailSender( + host=os.environ["SES_SMTP_HOST"], + port=int(os.environ["SES_SMTP_PORT"]), + username=os.environ["SES_SMTP_USERNAME"], + password=os.environ["SES_SMTP_PASSWORD"], + from_address=os.environ["SES_SMTP_FROM_ADDRESS"], + ) + + +@task_handler(task_source="bulk_document_download", source=Source.PORTFOLIO) +def handler(body: dict[str, Any], context: Any) -> dict[str, Any]: + trigger = BulkDocumentDownloadTriggerBody.model_validate(body) + engine = make_engine(PostgresConfig.from_env(os.environ)) + session = make_session(engine) + try: + subtask = SubTaskPostgresRepository(session).get(trigger.subtask_id) + recipe: dict[str, Any] = subtask.inputs or {} + landlord_property_ids: list[str] = [ + str(x) for x in recipe["landlord_property_ids"] + ] + recipient_email: str = str(recipe["recipient_email"]) + package_name: str = str(recipe["package_name"]) + + boto_s3: Any = boto3.client("s3") # pyright: ignore[reportUnknownMemberType] + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=UploadedFilePostgresRepository(session), + addresses=LandlordAddressResolver(session, landlord_property_ids), + documents=S3DocumentBytesReader(boto_s3), + packages=S3Client(boto_s3, os.environ["DOCUMENT_EXPORTS_BUCKET"]), + email=_email_sender(), + url_ttl_seconds=_URL_TTL_SECONDS, + ) + result = orchestrator.run( + landlord_property_ids=landlord_property_ids, + recipient_email=recipient_email, + package_name=package_name, + ) + finally: + session.close() + + return { + "presigned_url": result.presigned_url, + "package_s3_key": result.package_s3_key, + "included": result.included, + "skipped": [ + { + "landlord_property_id": skip.landlord_property_id, + "s3_key": skip.s3_key, + "reason": skip.reason, + } + for skip in result.skipped + ], + } diff --git a/applications/bulk_document_download/requirements.txt b/applications/bulk_document_download/requirements.txt new file mode 100644 index 000000000..6a85a2552 --- /dev/null +++ b/applications/bulk_document_download/requirements.txt @@ -0,0 +1,4 @@ +boto3 +pydantic +sqlmodel +psycopg2-binary