diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 228b1fcea..cc2bf4219 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -538,7 +538,7 @@ jobs: # Deploy FastAPI Lambda # ============================================================ fast_api_lambda: - needs: [determine_stage, ara_engine_lambda, categorisation_lambda, postcodeSplitter_lambda, bulk_address2uprn_combiner_lambda, bulkUploadFinaliser_lambda, modelling_e2e_lambda] + needs: [determine_stage, ara_engine_lambda, categorisation_lambda, postcodeSplitter_lambda, bulk_address2uprn_combiner_lambda, bulkUploadFinaliser_lambda, modelling_e2e_lambda, bulk_document_download_lambda] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: ara_fast_api @@ -816,6 +816,42 @@ jobs: TF_VAR_open_epc_api_token: ${{ secrets.DEV_OPEN_EPC_API_TOKEN }} TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + # ============================================================ + # Build Bulk Document Download image and Push + # ============================================================ + bulk_document_download_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: bulk-document-download-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: applications/bulk_document_download/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Bulk Document Download Lambda + # ============================================================ + bulk_document_download_lambda: + needs: [bulk_document_download_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: bulk_document_download + lambda_path: deployment/terraform/lambda/bulk_document_download + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: bulk-document-download-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.bulk_document_download_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + # ============================================================ # Deploy Hubspot ETL Lambda # ============================================================ diff --git a/deployment/terraform/lambda/bulk_document_download/main.tf b/deployment/terraform/lambda/bulk_document_download/main.tf new file mode 100644 index 000000000..3325a1ff1 --- /dev/null +++ b/deployment/terraform/lambda/bulk_document_download/main.tf @@ -0,0 +1,111 @@ +############################################ +# Credentials from Secrets Manager (resolved at plan/apply time and baked into +# env vars). The SES SMTP credentials are an IAM *user* (see modules/ses), so +# the Lambda authenticates to SES over SMTP with them — the execution role needs +# neither ses:* nor runtime secretsmanager access. +############################################ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +data "aws_secretsmanager_secret_version" "ses_smtp" { + secret_id = "${var.stage}/ses/smtp_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) + ses_smtp = jsondecode(data.aws_secretsmanager_secret_version.ses_smtp.secret_string) + + document_exports_bucket = "retrofit-document-exports-${var.stage}" + + # Source buckets the uploaded_files rows point at (read for packaging). + # OPEN QUESTION: confirm this is the closed set — if files can live in + # arbitrary buckets, widen to ["arn:aws:s3:::*"]. + source_bucket_arns = [ + "arn:aws:s3:::retrofit-data-${var.stage}", + "arn:aws:s3:::retrofit-energy-assessments-${var.stage}", + ] +} + +############################################ +# Lambda + SQS queue + trigger +############################################ +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = var.lambda_name + stage = var.stage + + image_uri = local.image_uri + + reserved_concurrent_executions = var.reserved_concurrent_executions + + batch_size = var.batch_size + maximum_concurrency = var.maximum_concurrency + + timeout = 900 + memory_size = 3008 + # A Download Package can be several GB; it is streamed to /tmp before the + # multipart upload (ADR-0060), so raise ephemeral storage to the 10 GB max. + ephemeral_storage_size = 10240 + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_HOST = var.db_host + POSTGRES_DATABASE = var.db_name + POSTGRES_PORT = var.db_port + + DOCUMENT_EXPORTS_BUCKET = local.document_exports_bucket + + # SES SMTP — IAM-user credentials sourced from Secrets Manager (modules/ses). + SES_SMTP_HOST = "email-smtp.eu-west-2.amazonaws.com" + SES_SMTP_PORT = "587" + SES_SMTP_USERNAME = local.ses_smtp.username + SES_SMTP_PASSWORD = local.ses_smtp.password + SES_SMTP_FROM_ADDRESS = var.ses_from_address + } +} + +############################################ +# IAM: read uploaded documents from their source buckets +############################################ +module "s3_read" { + source = "../../modules/s3_iam_policy" + + policy_name = "BulkDocumentDownloadS3Read-${var.stage}" + policy_description = "Allow bulk_document_download Lambda to read uploaded_files objects from their source buckets" + bucket_arns = local.source_bucket_arns + actions = ["s3:GetObject", "s3:ListBucket"] + resource_paths = ["/*"] +} + +resource "aws_iam_role_policy_attachment" "s3_read" { + role = module.lambda.role_name + policy_arn = module.s3_read.policy_arn +} + +############################################ +# IAM: write + read the assembled ZIP on the dedicated exports bucket. +# GetObject is required so the Lambda-signed presigned GET URL resolves. +############################################ +module "s3_exports" { + source = "../../modules/s3_iam_policy" + + policy_name = "BulkDocumentDownloadS3Exports-${var.stage}" + policy_description = "Allow bulk_document_download Lambda to write and presign-read Download Packages on the exports bucket" + bucket_arns = ["arn:aws:s3:::${local.document_exports_bucket}"] + actions = ["s3:PutObject", "s3:GetObject"] + resource_paths = ["/*"] +} + +resource "aws_iam_role_policy_attachment" "s3_exports" { + role = module.lambda.role_name + policy_arn = module.s3_exports.policy_arn +} + +# NOTE: no ses:* on the role — the handler sends over SMTP with the SES IAM-user +# credentials injected above. diff --git a/deployment/terraform/lambda/bulk_document_download/outputs.tf b/deployment/terraform/lambda/bulk_document_download/outputs.tf new file mode 100644 index 000000000..7f2442c26 --- /dev/null +++ b/deployment/terraform/lambda/bulk_document_download/outputs.tf @@ -0,0 +1,9 @@ +output "bulk_document_download_queue_url" { + value = module.lambda.queue_url + description = "URL of the bulk-document-download SQS queue (FastAPI enqueues jobs here)" +} + +output "bulk_document_download_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the bulk-document-download SQS queue" +} diff --git a/deployment/terraform/lambda/bulk_document_download/provider.tf b/deployment/terraform/lambda/bulk_document_download/provider.tf new file mode 100644 index 000000000..2ea6c350c --- /dev/null +++ b/deployment/terraform/lambda/bulk_document_download/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "bulk-document-download-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "eu-west-2" +} diff --git a/deployment/terraform/lambda/bulk_document_download/variables.tf b/deployment/terraform/lambda/bulk_document_download/variables.tf new file mode 100644 index 000000000..9abb25362 --- /dev/null +++ b/deployment/terraform/lambda/bulk_document_download/variables.tf @@ -0,0 +1,61 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "reserved_concurrent_executions" { + type = number + default = -1 + description = "Reserved concurrency for the Lambda. -1 = unreserved." +} + +variable "maximum_concurrency" { + type = number + default = 5 + description = "Maximum concurrent Lambda invocations from the SQS trigger." +} + +variable "batch_size" { + type = number + default = 1 +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} + +variable "ses_from_address" { + type = string + description = "Verified SES sender for Download Package notifications (ADR-0059)." + default = "noreply@domna.homes" +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} diff --git a/deployment/terraform/lambda/fast-api/main.tf b/deployment/terraform/lambda/fast-api/main.tf index 26e154d6a..73310b999 100644 --- a/deployment/terraform/lambda/fast-api/main.tf +++ b/deployment/terraform/lambda/fast-api/main.tf @@ -73,6 +73,15 @@ data "terraform_remote_state" "modelling_e2e" { } } +data "terraform_remote_state" "bulk_document_download" { + backend = "s3" + config = { + bucket = "bulk-document-download-terraform-state", + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + ############################################ # Load Credentials ############################################ @@ -136,6 +145,8 @@ module "fastapi" { FINALISER_SQS_URL = data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_url LANDLORD_OVERRIDES_SQS_URL = data.terraform_remote_state.landlord_description_overrides.outputs.landlord_description_overrides_queue_url MODELLING_E2E_SQS_URL = data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_url + + BULK_DOCUMENT_DOWNLOAD_SQS_URL = data.terraform_remote_state.bulk_document_download.outputs.bulk_document_download_queue_url } } @@ -160,7 +171,8 @@ module "fastapi_sqs_policy" { data.terraform_remote_state.bulk_address2uprn_combiner.outputs.bulk_address2uprn_combiner_queue_arn, data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_arn, data.terraform_remote_state.landlord_description_overrides.outputs.landlord_description_overrides_queue_arn, - data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_arn + data.terraform_remote_state.modelling_e2e.outputs.modelling_e2e_queue_arn, + data.terraform_remote_state.bulk_document_download.outputs.bulk_document_download_queue_arn ] conditions = null diff --git a/deployment/terraform/modules/lambda_service/main.tf b/deployment/terraform/modules/lambda_service/main.tf index 3250110ba..0981deac1 100644 --- a/deployment/terraform/modules/lambda_service/main.tf +++ b/deployment/terraform/modules/lambda_service/main.tf @@ -11,6 +11,10 @@ resource "aws_lambda_function" "this" { reserved_concurrent_executions = var.reserved_concurrent_executions + ephemeral_storage { + size = var.ephemeral_storage_size + } + environment { variables = var.environment } diff --git a/deployment/terraform/modules/lambda_service/variables.tf b/deployment/terraform/modules/lambda_service/variables.tf index 46241f300..c0ddc10ff 100644 --- a/deployment/terraform/modules/lambda_service/variables.tf +++ b/deployment/terraform/modules/lambda_service/variables.tf @@ -12,6 +12,12 @@ variable "memory_size" { default = 512 } +variable "ephemeral_storage_size" { + type = number + default = 512 + description = "Lambda /tmp size in MB (512-10240). Default 512 keeps every existing Lambda unchanged; raised only where a large on-disk artifact is built (e.g. multi-GB Download Package ZIPs, ADR-0060)." +} + variable "environment" { type = map(string) default = {} diff --git a/deployment/terraform/modules/lambda_with_sqs/main.tf b/deployment/terraform/modules/lambda_with_sqs/main.tf index 97f867935..2d8259dcf 100644 --- a/deployment/terraform/modules/lambda_with_sqs/main.tf +++ b/deployment/terraform/modules/lambda_with_sqs/main.tf @@ -31,6 +31,8 @@ module "lambda" { timeout = var.timeout memory_size = var.memory_size + ephemeral_storage_size = var.ephemeral_storage_size + environment = var.environment reserved_concurrent_executions = var.reserved_concurrent_executions } diff --git a/deployment/terraform/modules/lambda_with_sqs/variables.tf b/deployment/terraform/modules/lambda_with_sqs/variables.tf index 90585e929..d75ff28e8 100644 --- a/deployment/terraform/modules/lambda_with_sqs/variables.tf +++ b/deployment/terraform/modules/lambda_with_sqs/variables.tf @@ -25,6 +25,11 @@ variable "memory_size" { default = 1024 } +variable "ephemeral_storage_size" { + type = number + default = 512 +} + variable "environment" { type = map(string) default = {} diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index b4cdafbcc..a492401de 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -895,6 +895,38 @@ output "modelling_e2e_ecr_url" { } +################################################ +# Bulk Document Download – Lambda (ADR-0060) +################################################ +module "bulk_document_download_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "bulk-document-download-terraform-state" +} + +module "bulk_document_download_registry" { + source = "../modules/container_registry" + name = "bulk-document-download" + stage = var.stage +} + +# Dedicated bucket for generated Download Packages — kept out of the shared data +# bucket so retention/IAM stay scoped to exports (ADR-0060). +module "document_exports" { + source = "../modules/s3" + bucketname = "retrofit-document-exports-${var.stage}" + allowed_origins = var.allowed_origins +} + +output "document_exports_bucket_name" { + value = module.document_exports.bucket_name + description = "Name of the document exports bucket (Download Packages)" +} + +output "bulk_document_download_ecr_url" { + value = module.bulk_document_download_registry.repository_url +} + + ################################################ # Abri OpenHousing – Lambda ################################################