Terraform + CI for bulk_document_download: exports bucket, 10GB /tmp, SES SMTP, wiring 🟩

New lambda_with_sqs consumer (timeout 900, memory 3008, ephemeral_storage 10240
for multi-GB ZIPs); dedicated retrofit-document-exports bucket (no lifecycle on
DATA_BUCKET); IAM to read source buckets + write/presign-read the exports
bucket; SES SMTP creds baked from Secrets Manager (no ses:* on the role). Adds
ephemeral_storage_size knob to the shared lambda modules (default 512, backward
compatible). Wires the queue url+arn into fast-api and orders the CI jobs
(ADR-0055).

NOTE: terraform is drafted, not validated (no AWS/terraform in the dev env).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/deploy_terraform.yml

@@ -538,7 +538,7 @@ jobs:
   # Deploy FastAPI Lambda
   # ============================================================
   fast_api_lambda:
-    needs: [determine_stage, ara_engine_lambda, categorisation_lambda, postcodeSplitter_lambda, bulk_address2uprn_combiner_lambda, bulkUploadFinaliser_lambda, modelling_e2e_lambda]
+    needs: [determine_stage, ara_engine_lambda, categorisation_lambda, postcodeSplitter_lambda, bulk_address2uprn_combiner_lambda, bulkUploadFinaliser_lambda, modelling_e2e_lambda, bulk_document_download_lambda]
     uses: ./.github/workflows/_deploy_lambda.yml
     with:
       lambda_name: ara_fast_api
@@ -816,6 +816,42 @@ jobs:
       TF_VAR_open_epc_api_token: ${{ secrets.DEV_OPEN_EPC_API_TOKEN }}
       TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
 
+  # ============================================================
+  # Build Bulk Document Download image and Push
+  # ============================================================
+  bulk_document_download_image:
+    needs: [determine_stage, shared_terraform]
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      ecr_repo: bulk-document-download-${{ needs.determine_stage.outputs.stage }}
+      dockerfile_path: applications/bulk_document_download/Dockerfile
+      build_context: .
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+
+  # ============================================================
+  # Deploy Bulk Document Download Lambda
+  # ============================================================
+  bulk_document_download_lambda:
+    needs: [bulk_document_download_image, determine_stage]
+    uses: ./.github/workflows/_deploy_lambda.yml
+    with:
+      lambda_name: bulk_document_download
+      lambda_path: deployment/terraform/lambda/bulk_document_download
+      stage: ${{ needs.determine_stage.outputs.stage }}
+      ecr_repo: bulk-document-download-${{ needs.determine_stage.outputs.stage }}
+      image_digest: ${{ needs.bulk_document_download_image.outputs.image_digest }}
+      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+      TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
+      TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
+      TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
+
   # ============================================================
   # Deploy Hubspot ETL Lambda
   # ============================================================

deployment/terraform/lambda/bulk_document_download/main.tf

@@ -0,0 +1,111 @@
+############################################
+# Credentials from Secrets Manager (resolved at plan/apply time and baked into
+# env vars). The SES SMTP credentials are an IAM *user* (see modules/ses), so
+# the Lambda authenticates to SES over SMTP with them — the execution role needs
+# neither ses:* nor runtime secretsmanager access.
+############################################
+data "aws_secretsmanager_secret_version" "db_credentials" {
+  secret_id = "${var.stage}/assessment_model/db_credentials"
+}
+
+data "aws_secretsmanager_secret_version" "ses_smtp" {
+  secret_id = "${var.stage}/ses/smtp_credentials"
+}
+
+locals {
+  db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
+  ses_smtp       = jsondecode(data.aws_secretsmanager_secret_version.ses_smtp.secret_string)
+
+  document_exports_bucket = "retrofit-document-exports-${var.stage}"
+
+  # Source buckets the uploaded_files rows point at (read for packaging).
+  # OPEN QUESTION: confirm this is the closed set — if files can live in
+  # arbitrary buckets, widen to ["arn:aws:s3:::*"].
+  source_bucket_arns = [
+    "arn:aws:s3:::retrofit-data-${var.stage}",
+    "arn:aws:s3:::retrofit-energy-assessments-${var.stage}",
+  ]
+}
+
+############################################
+# Lambda + SQS queue + trigger
+############################################
+module "lambda" {
+  source = "../../modules/lambda_with_sqs"
+
+  name  = var.lambda_name
+  stage = var.stage
+
+  image_uri = local.image_uri
+
+  reserved_concurrent_executions = var.reserved_concurrent_executions
+
+  batch_size          = var.batch_size
+  maximum_concurrency = var.maximum_concurrency
+
+  timeout     = 900
+  memory_size = 3008
+  # A Download Package can be several GB; it is streamed to /tmp before the
+  # multipart upload (ADR-0060), so raise ephemeral storage to the 10 GB max.
+  ephemeral_storage_size = 10240
+
+  environment = {
+    STAGE     = var.stage
+    LOG_LEVEL = "info"
+
+    POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username
+    POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password
+    POSTGRES_HOST     = var.db_host
+    POSTGRES_DATABASE = var.db_name
+    POSTGRES_PORT     = var.db_port
+
+    DOCUMENT_EXPORTS_BUCKET = local.document_exports_bucket
+
+    # SES SMTP — IAM-user credentials sourced from Secrets Manager (modules/ses).
+    SES_SMTP_HOST         = "email-smtp.eu-west-2.amazonaws.com"
+    SES_SMTP_PORT         = "587"
+    SES_SMTP_USERNAME     = local.ses_smtp.username
+    SES_SMTP_PASSWORD     = local.ses_smtp.password
+    SES_SMTP_FROM_ADDRESS = var.ses_from_address
+  }
+}
+
+############################################
+# IAM: read uploaded documents from their source buckets
+############################################
+module "s3_read" {
+  source = "../../modules/s3_iam_policy"
+
+  policy_name        = "BulkDocumentDownloadS3Read-${var.stage}"
+  policy_description = "Allow bulk_document_download Lambda to read uploaded_files objects from their source buckets"
+  bucket_arns        = local.source_bucket_arns
+  actions            = ["s3:GetObject", "s3:ListBucket"]
+  resource_paths     = ["/*"]
+}
+
+resource "aws_iam_role_policy_attachment" "s3_read" {
+  role       = module.lambda.role_name
+  policy_arn = module.s3_read.policy_arn
+}
+
+############################################
+# IAM: write + read the assembled ZIP on the dedicated exports bucket.
+# GetObject is required so the Lambda-signed presigned GET URL resolves.
+############################################
+module "s3_exports" {
+  source = "../../modules/s3_iam_policy"
+
+  policy_name        = "BulkDocumentDownloadS3Exports-${var.stage}"
+  policy_description = "Allow bulk_document_download Lambda to write and presign-read Download Packages on the exports bucket"
+  bucket_arns        = ["arn:aws:s3:::${local.document_exports_bucket}"]
+  actions            = ["s3:PutObject", "s3:GetObject"]
+  resource_paths     = ["/*"]
+}
+
+resource "aws_iam_role_policy_attachment" "s3_exports" {
+  role       = module.lambda.role_name
+  policy_arn = module.s3_exports.policy_arn
+}
+
+# NOTE: no ses:* on the role — the handler sends over SMTP with the SES IAM-user
+# credentials injected above.

deployment/terraform/lambda/bulk_document_download/outputs.tf

@@ -0,0 +1,9 @@
+output "bulk_document_download_queue_url" {
+  value       = module.lambda.queue_url
+  description = "URL of the bulk-document-download SQS queue (FastAPI enqueues jobs here)"
+}
+
+output "bulk_document_download_queue_arn" {
+  value       = module.lambda.queue_arn
+  description = "ARN of the bulk-document-download SQS queue"
+}

deployment/terraform/lambda/bulk_document_download/provider.tf

@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+  }
+
+  backend "s3" {
+    bucket = "bulk-document-download-terraform-state"
+    key    = "terraform.tfstate"
+    region = "eu-west-2"
+  }
+
+  required_version = ">= 1.2.0"
+}
+
+provider "aws" {
+  region = "eu-west-2"
+}

deployment/terraform/lambda/bulk_document_download/variables.tf

@@ -0,0 +1,61 @@
+variable "lambda_name" {
+  type        = string
+  description = "Logical name of the lambda"
+}
+
+variable "stage" {
+  description = "Deployment stage (e.g. dev, prod)"
+  type        = string
+}
+
+variable "ecr_repo_url" {
+  type        = string
+  description = "ECR repository URL (no tag, no digest)"
+}
+
+variable "image_digest" {
+  type        = string
+  description = "Image digest (sha256:...)"
+}
+
+variable "reserved_concurrent_executions" {
+  type        = number
+  default     = -1
+  description = "Reserved concurrency for the Lambda. -1 = unreserved."
+}
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = 5
+  description = "Maximum concurrent Lambda invocations from the SQS trigger."
+}
+
+variable "batch_size" {
+  type    = number
+  default = 1
+}
+
+variable "db_host" {
+  type      = string
+  sensitive = true
+}
+
+variable "db_name" {
+  type      = string
+  sensitive = true
+}
+
+variable "db_port" {
+  type      = string
+  sensitive = true
+}
+
+variable "ses_from_address" {
+  type        = string
+  description = "Verified SES sender for Download Package notifications (ADR-0059)."
+  default     = "noreply@domna.homes"
+}
+
+locals {
+  image_uri = "${var.ecr_repo_url}@${var.image_digest}"
+}

deployment/terraform/lambda/fast-api/main.tf

@@ -73,6 +73,15 @@ data "terraform_remote_state" "modelling_e2e" {
   }
 }
 
+data "terraform_remote_state" "bulk_document_download" {
+  backend = "s3"
+  config = {
+    bucket = "bulk-document-download-terraform-state",
+    key    = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+
 ############################################
 # Load Credentials
 ############################################
@@ -136,6 +145,8 @@ module "fastapi" {
     FINALISER_SQS_URL          = data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_url
     LANDLORD_OVERRIDES_SQS_URL = data.terraform_remote_state.landlord_description_overrides.outputs.landlord_description_overrides_queue_url
     MODELLING_E2E_SQS_URL      = data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_url
+
+    BULK_DOCUMENT_DOWNLOAD_SQS_URL = data.terraform_remote_state.bulk_document_download.outputs.bulk_document_download_queue_url
   }
 }
 
@@ -160,7 +171,8 @@ module "fastapi_sqs_policy" {
     data.terraform_remote_state.bulk_address2uprn_combiner.outputs.bulk_address2uprn_combiner_queue_arn,
     data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_arn,
     data.terraform_remote_state.landlord_description_overrides.outputs.landlord_description_overrides_queue_arn,
-    data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_arn
+    data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_arn,
+    data.terraform_remote_state.bulk_document_download.outputs.bulk_document_download_queue_arn
   ]
 
   conditions = null

deployment/terraform/modules/lambda_service/main.tf

@@ -11,6 +11,10 @@ resource "aws_lambda_function" "this" {
 
   reserved_concurrent_executions = var.reserved_concurrent_executions
 
+  ephemeral_storage {
+    size = var.ephemeral_storage_size
+  }
+
   environment {
     variables = var.environment
   }

deployment/terraform/modules/lambda_service/variables.tf

@@ -12,6 +12,12 @@ variable "memory_size" {
   default = 512
 }
 
+variable "ephemeral_storage_size" {
+  type        = number
+  default     = 512
+  description = "Lambda /tmp size in MB (512-10240). Default 512 keeps every existing Lambda unchanged; raised only where a large on-disk artifact is built (e.g. multi-GB Download Package ZIPs, ADR-0060)."
+}
+
 variable "environment" {
   type    = map(string)
   default = {}

deployment/terraform/modules/lambda_with_sqs/main.tf

@@ -31,6 +31,8 @@ module "lambda" {
   timeout     = var.timeout
   memory_size = var.memory_size
 
+  ephemeral_storage_size = var.ephemeral_storage_size
+
   environment                     = var.environment
   reserved_concurrent_executions  = var.reserved_concurrent_executions
 }

deployment/terraform/modules/lambda_with_sqs/variables.tf

@@ -25,6 +25,11 @@ variable "memory_size" {
   default = 1024
 }
 
+variable "ephemeral_storage_size" {
+  type    = number
+  default = 512
+}
+
 variable "environment" {
   type    = map(string)
   default = {}

deployment/terraform/shared/main.tf

@@ -895,6 +895,38 @@ output "modelling_e2e_ecr_url" {
 }
 
 
+################################################
+# Bulk Document Download – Lambda (ADR-0060)
+################################################
+module "bulk_document_download_state_bucket" {
+  source      = "../modules/tf_state_bucket"
+  bucket_name = "bulk-document-download-terraform-state"
+}
+
+module "bulk_document_download_registry" {
+  source = "../modules/container_registry"
+  name   = "bulk-document-download"
+  stage  = var.stage
+}
+
+# Dedicated bucket for generated Download Packages — kept out of the shared data
+# bucket so retention/IAM stay scoped to exports (ADR-0060).
+module "document_exports" {
+  source          = "../modules/s3"
+  bucketname      = "retrofit-document-exports-${var.stage}"
+  allowed_origins = var.allowed_origins
+}
+
+output "document_exports_bucket_name" {
+  value       = module.document_exports.bucket_name
+  description = "Name of the document exports bucket (Download Packages)"
+}
+
+output "bulk_document_download_ecr_url" {
+  value = module.bulk_document_download_registry.repository_url
+}
+
+
 ################################################
 # Abri OpenHousing – Lambda
 ################################################