From 54255942f93acaf71521cf9d7d6c4e9baf2459fb Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 08:51:08 +0000 Subject: [PATCH 01/16] add lambda_service_zip and lambda_with_api_gateway terraform modules --- .github/workflows/deploy_terraform.yml | 14 +++++------ .../modules/lambda_service_zip/main.tf | 24 +++++++++++++++++++ .../modules/lambda_with_api_gateway/main.tf | 0 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 infrastructure/terraform/modules/lambda_service_zip/main.tf create mode 100644 infrastructure/terraform/modules/lambda_with_api_gateway/main.tf diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index bde7eb21..042bce35 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -41,7 +41,7 @@ jobs: fi # ============================================================ - # 1️⃣ Shared Terraform (infra) + # Shared Terraform (infra) # ============================================================ shared_terraform: needs: determine_stage @@ -79,7 +79,7 @@ jobs: run: terraform apply -auto-approve tfplan # ============================================================ - # 2️⃣ Build Address 2 UPRN image and Push + # Build Address 2 UPRN image and Push # ============================================================ address2uprn_image: needs: [determine_stage, shared_terraform] @@ -103,7 +103,7 @@ jobs: EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }} # ============================================================ - # 3️⃣ Deploy Address 2 UPRN Lambda + # Deploy Address 2 UPRN Lambda # ============================================================ address2uprn_lambda: needs: [address2uprn_image, determine_stage] @@ -122,7 +122,7 @@ jobs: # ============================================================ - # 2️⃣ Build Postcode Splitter image and Push + # Build Postcode Splitter image and Push # ============================================================ postcodeSplitter_image: needs: [determine_stage, shared_terraform] @@ -144,7 +144,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy Postcode Splitter Lambda + # Deploy Postcode Splitter Lambda # ============================================================ postcodeSplitter_lambda: needs: [postcodeSplitter_image, determine_stage, address2uprn_lambda] @@ -283,7 +283,7 @@ jobs: TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} # ============================================================ - # 2️⃣ Build OrdanceSurvey image and Push + # Build OrdanceSurvey image and Push # ============================================================ ordnanceSurvey_image: needs: [determine_stage, shared_terraform] @@ -305,7 +305,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy OrdanceSurvey Lambda + # Deploy OrdanceSurvey Lambda # ============================================================ ordnanceSurvey_lambda: needs: [ordnanceSurvey_image, determine_stage] diff --git a/infrastructure/terraform/modules/lambda_service_zip/main.tf b/infrastructure/terraform/modules/lambda_service_zip/main.tf new file mode 100644 index 00000000..285aa9d4 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service_zip/main.tf @@ -0,0 +1,24 @@ +resource "aws_lambda_function" "this" { + function_name = var.name + role = var.role_arn + package_type = "Zip" + filename = var.filename + source_code_hash = var.source_code_hash + handler = var.handler + runtime = var.runtime + timeout = var.timeout + memory_size = var.memory_size + publish = true + + environment { + variables = var.environment + } +} + +output "lambda_arn" { + value = aws_lambda_function.this.arn +} + +output "function_name" { + value = aws_lambda_function.this.function_name +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf new file mode 100644 index 00000000..e69de29b From db50f8613f5223d11b0a40ff20f73fc0a158e23a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 08:58:40 +0000 Subject: [PATCH 02/16] make ecr_repo_url and image_digest optional in _deploy_lambda.yml --- .github/workflows/_deploy_lambda.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index ce1a0e77..7e11cf32 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -16,12 +16,14 @@ on: type: string ecr_repo: - required: true + required: false type: string + default: '' image_digest: - required: true + required: false type: string + default: '' terraform_apply: required: false @@ -116,11 +118,15 @@ jobs: TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} run: | + EXTRA_VARS="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}" + fi + terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" \ + $EXTRA_VARS \ -out=lambdaplan - name: Terraform Apply @@ -141,8 +147,12 @@ jobs: TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} run: | + EXTRA_VARS="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}" + fi + terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" + $EXTRA_VARS From 3423ca76fb4adb137eb57825ab66112731c28f9f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 10:00:08 +0000 Subject: [PATCH 03/16] define terraform for lambda_with_api_gateway and lambda_service_zip --- .../modules/lambda_service_zip/variables.tf | 9 ++ .../modules/lambda_with_api_gateway/main.tf | 103 ++++++++++++++++++ .../lambda_with_api_gateway/outputs.tf | 11 ++ .../lambda_with_api_gateway/variables.tf | 18 +++ 4 files changed, 141 insertions(+) create mode 100644 infrastructure/terraform/modules/lambda_service_zip/variables.tf create mode 100644 infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf create mode 100644 infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf diff --git a/infrastructure/terraform/modules/lambda_service_zip/variables.tf b/infrastructure/terraform/modules/lambda_service_zip/variables.tf new file mode 100644 index 00000000..3a243c49 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service_zip/variables.tf @@ -0,0 +1,9 @@ +variable "name" { type = string } +variable "role_arn" { type = string } +variable "filename" { type = string } +variable "source_code_hash" { type = string } +variable "handler" { type = string } +variable "runtime" { type = string } +variable "timeout" { type = number default = 30 } +variable "memory_size" { type = number default = 128 } +variable "environment" { type = map(string) default = {} } \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf index e69de29b..23ccc4b1 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf @@ -0,0 +1,103 @@ +############################################ +# IAM role +############################################ +module "role" { + source = "../lambda_execution_role" + name = "${var.name}-lambda-${var.stage}" +} + +############################################ +# Zip the source code +############################################ +data "archive_file" "this" { + type = "zip" + source_dir = var.source_dir + output_path = "${path.module}/lambda_package.zip" + excludes = var.zip_excludes +} + +############################################ +# Lambda +############################################ +module "lambda" { + source = "../lambda_service_zip" + + name = "${var.name}-${var.stage}" + role_arn = module.role.role_arn + filename = data.archive_file.this.output_path + source_code_hash = data.archive_file.this.output_base64sha256 + handler = var.handler + runtime = var.runtime + timeout = var.timeout + memory_size = var.memory_size + environment = var.environment +} + +############################################ +# API Gateway +############################################ +resource "aws_apigatewayv2_api" "this" { + name = "${var.name}-api-${var.stage}" + protocol_type = "HTTP" +} + +resource "aws_apigatewayv2_stage" "this" { + api_id = aws_apigatewayv2_api.this.id + name = "$default" + auto_deploy = true +} + +resource "aws_apigatewayv2_integration" "this" { + api_id = aws_apigatewayv2_api.this.id + integration_type = "AWS_PROXY" + integration_uri = module.lambda.lambda_arn + payload_format_version = "2.0" +} + +resource "aws_apigatewayv2_route" "catch_all" { + api_id = aws_apigatewayv2_api.this.id + route_key = "$default" + target = "integrations/${aws_apigatewayv2_integration.this.id}" +} + +resource "aws_lambda_permission" "apigw_invoke" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = module.lambda.lambda_arn + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.this.execution_arn}/*/*" +} + +############################################ +# Custom domain +############################################ +resource "aws_apigatewayv2_domain_name" "this" { + count = var.domain_name != null ? 1 : 0 + domain_name = var.domain_name + + domain_name_configuration { + certificate_arn = var.certificate_arn + endpoint_type = "REGIONAL" + security_policy = "TLS_1_2" + } +} + +resource "aws_apigatewayv2_api_mapping" "this" { + count = var.domain_name != null ? 1 : 0 + api_id = aws_apigatewayv2_api.this.id + domain_name = aws_apigatewayv2_domain_name.this[0].id + stage = aws_apigatewayv2_stage.this.id +} + +resource "aws_route53_record" "this" { + count = var.domain_name != null ? 1 : 0 + name = aws_apigatewayv2_domain_name.this[0].domain_name + type = "A" + zone_id = var.route53_zone_id + + alias { + name = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].target_domain_name + zone_id = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].hosted_zone_id + evaluate_target_health = false + } +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf new file mode 100644 index 00000000..52db1ff9 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf @@ -0,0 +1,11 @@ +output "role_name" { + value = module.role.role_name +} + +output "api_endpoint" { + value = aws_apigatewayv2_stage.this.invoke_url +} + +output "custom_domain_endpoint" { + value = var.domain_name != null ? "https://${var.domain_name}" : null +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf new file mode 100644 index 00000000..1a08ff2e --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf @@ -0,0 +1,18 @@ +variable "name" { type = string } +variable "stage" { type = string } +variable "source_dir" { type = string } +variable "handler" { type = string } +variable "runtime" { type = string } + +variable "zip_excludes" { + type = list(string) + default = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**"] +} + +variable "timeout" { type = number default = 600 } +variable "memory_size" { type = number default = 512 } +variable "environment" { type = map(string) default = {} } + +variable "domain_name" { type = string default = null } +variable "certificate_arn" { type = string default = null } +variable "route53_zone_id" { type = string default = null } \ No newline at end of file From dcb3efdb7ed97fdd2818cccd8986551f5d88ccbd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 11:42:06 +0000 Subject: [PATCH 04/16] update fast-api terraform --- .../terraform/lambda/fast-api/main.tf | 122 +++++++++++++----- .../terraform/lambda/fast-api/provider.tf | 2 +- .../terraform/lambda/fast-api/variables.tf | 47 ++++--- 3 files changed, 118 insertions(+), 53 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index 104d4a4d..aabc1c52 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -7,43 +7,101 @@ data "terraform_remote_state" "shared" { } } -module "lambda" { - source = "../../modules/lambda_with_sqs" +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} - name = REPLACE ME #"address2uprn" for example - stage = var.stage +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} - image_uri = local.image_uri - # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) - maximum_concurrency = var.maximum_concurrency +data "aws_ssm_parameter" "certificate_arn" { + name = "/ssl_certificate_arn" +} - batch_size = var.batch_size +data "aws_route53_zone" "this" { + name = var.domain_name +} - environment = { - STAGE = var.stage - LOG_LEVEL = "info" +############################################ +# Install Python requirements +############################################ +resource "null_resource" "pip_install" { + triggers = { + requirements_hash = filemd5("${path.root}/../../../../backend/app/requirements/requirements.txt") + } + + provisioner "local-exec" { + command = < Date: Tue, 10 Mar 2026 12:04:53 +0000 Subject: [PATCH 05/16] read queue names from terraform state --- .../lambda/categorisation/outputs.tf | 9 ++++++++ .../terraform/lambda/engine/outputs.tf | 9 ++++++++ .../terraform/lambda/fast-api/main.tf | 22 +++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 infrastructure/terraform/lambda/categorisation/outputs.tf create mode 100644 infrastructure/terraform/lambda/engine/outputs.tf diff --git a/infrastructure/terraform/lambda/categorisation/outputs.tf b/infrastructure/terraform/lambda/categorisation/outputs.tf new file mode 100644 index 00000000..be1ac118 --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/outputs.tf @@ -0,0 +1,9 @@ +output "categorisation_queue_url" { + value = module.lambda.queu_url + description = "URL of the Categorisation SQS queue" +} + +output "categorisation_queue_arn" { + value = module.lambda.queu_arn + description = "ARN of the Categorisation SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/outputs.tf b/infrastructure/terraform/lambda/engine/outputs.tf new file mode 100644 index 00000000..bba2046b --- /dev/null +++ b/infrastructure/terraform/lambda/engine/outputs.tf @@ -0,0 +1,9 @@ +output "ara_engine_queue_url" { + value = module.lambda.queu_url + description = "URL of the Engine SQS queue" +} + +output "ara_engine_queue_arn" { + value = module.lambda.queu_arn + description = "ARN of the Engine SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index aabc1c52..82d2cc60 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -7,6 +7,24 @@ data "terraform_remote_state" "shared" { } } +data "terraform_remote_state" "engine" { + backend = "s3" + config = { + bucket = "ara-engine-terraform-state", + key = "env:/${var.stage}/teraform.tfstate" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "categorisation" { + backend = "s3" + config = { + bucket = "categorisation-terraform-state", + key = "env:/${var.stage}/teraform.tfstate" + region = "eu-west-2" + } +} + data "aws_secretsmanager_secret_version" "db_credentials" { secret_id = "${var.stage}/assessment_model/db_credentials" } @@ -88,8 +106,8 @@ module "fastapi" { HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name - ENGINE_SQS_URL = data.terraform_remote_state.shared.outputs.engine_queue_url - CATEGORISATION_SQS_URL = "https://sqs.eu-west-2.amazonaws.com/337213553626/categorisation-queue-dev" + ENGINE_SQS_URL = data.terraform_remote_state.engine.ara_engine_queue_url + CATEGORISATION_SQS_URL = data.terraform_remote_state.categorisation.categorisation_queue_url } } From 783ddd7be008120bbc9a9cd0d2cf23b9ec0c8bbe Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 12:09:52 +0000 Subject: [PATCH 06/16] add some comments --- infrastructure/terraform/lambda/fast-api/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index 82d2cc60..ebf436c3 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -1,3 +1,6 @@ +############################################ +# Load Terraform State +############################################ data "terraform_remote_state" "shared" { backend = "s3" config = { @@ -25,6 +28,9 @@ data "terraform_remote_state" "categorisation" { } } +############################################ +# Load Credentials +############################################ data "aws_secretsmanager_secret_version" "db_credentials" { secret_id = "${var.stage}/assessment_model/db_credentials" } From 891ccd4a8b45f9239b854ff648e914b37feb9134 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 13:55:45 +0000 Subject: [PATCH 07/16] fast api s3 policy --- .../terraform/lambda/fast-api/main.tf | 4 +-- infrastructure/terraform/shared/main.tf | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index ebf436c3..40e0f4f9 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -122,10 +122,10 @@ module "fastapi" { ############################################ resource "aws_iam_role_policy_attachment" "fastapi_s3_read" { role = module.fastapi.role_name - policy_arn = data.terraform_remote_state.shared.outputs.fastapi_s3_read_arn + policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_arn } resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" { role = module.fastapi.role_name - policy_arn = data.terraform_remote_state.shared.outputs.fastapi_sqs_send_arn + policy_arn = data.terraform_remote_state.shared.outputs.fast_api_sqs_send_arn } \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index eaacddec..f4b2cd3f 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -535,3 +535,29 @@ module "ara_fastapi_registry" { name = "ara-fastapi" stage = var.stage } + +# S3 policy for FastAPI app to read and write from various S3 buckets +module "fast_api_s3_read_and_write" { + source = "../modules/s3_iam_policy" + + policy_name = "FastAPIReadandWriteS3" + policy_description = "Allow FastAPI Lambda to read from and write to various S3 buckets" + bucket_arns = [ + "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}" + ] + actions = ["s3:GetObject", "s3:ListBucket"] + resource_paths = ["/*"] +} + +output "fast_api_s3_read_and_write_arn" { + value = module.fast_api_s3_read_and_write.policy_arn +} + + From f3d51c4c7c0d7bb19101ca93c03fa69f8f2578d6 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 14:59:46 +0000 Subject: [PATCH 08/16] sqs permissions --- .../terraform/lambda/fast-api/main.tf | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index 40e0f4f9..cb4c923d 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -120,12 +120,36 @@ module "fastapi" { ############################################ # IAM policy attachments ############################################ -resource "aws_iam_role_policy_attachment" "fastapi_s3_read" { +resource "aws_iam_role_policy_attachment" "fast_api_s3_read" { role = module.fastapi.role_name policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_arn } +module "fastapi_sqs_policy" { + source = "../../modules/generic_iam_policy" + + policy_name = "fastapi-sqs-send-${var.stage}" + policy_description = "Allow FastAPI to send messages to engine & categorisation queues" + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + data.terraform_remote_state.engine.outputs.ara_engine_queue_arn, + data.terraform_remote_state.categorisation.outputs.categorisation_queue_arn + ] + + conditions = null + + tags = { + Service = "fastapi" + Stage = var.stage + } +} + + resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" { role = module.fastapi.role_name - policy_arn = data.terraform_remote_state.shared.outputs.fast_api_sqs_send_arn + policy_arn = module.fastapi_sqs_policy.policy_arn } \ No newline at end of file From 9c4a8e11dba8f8a1644abb56be88603b3660d9a3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 15:15:21 +0000 Subject: [PATCH 09/16] correct s3 policy attachment --- infrastructure/terraform/lambda/fast-api/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index cb4c923d..e339b582 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -149,7 +149,7 @@ module "fastapi_sqs_policy" { } -resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" { +resource "aws_iam_role_policy_attachment" "fastapi_sqs_read_and_write" { role = module.fastapi.role_name - policy_arn = module.fastapi_sqs_policy.policy_arn + policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_and_write_arn } \ No newline at end of file From 201888bc3a05d3c5daee24ab289a7fd80cb00a38 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 16:24:10 +0000 Subject: [PATCH 10/16] addres JTK PR: split optional ecr uri and image digest logic in deploy lambda --- .github/workflows/_deploy_lambda.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 420f9504..3cef705e 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -121,15 +121,21 @@ jobs: TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | - EXTRA_VARS="" + ECR_REPO_URL_VAR="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then - EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}" + ECR_REPO_URL_VAR="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" + fi + + IMAGE_DIGEST_VAR="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + IMAGE_DIGEST_VAR="-var=image_digest=${{ inputs.image_digest }}" fi terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - $EXTRA_VARS \ + $ECR_REPO_URL_VAR \ + $IMAGE_DIGEST_VAR \ -out=lambdaplan - name: Terraform Apply From 510e2736340af049a2928e606102e8f338d04eb0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Mar 2026 16:30:27 +0000 Subject: [PATCH 11/16] addres JTK PR: remove pip upgrade after installing requirements --- infrastructure/terraform/lambda/fast-api/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index e339b582..c87897c7 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -65,7 +65,6 @@ resource "null_resource" "pip_install" { --implementation cp \ --python-version 3.11 \ --only-binary=:all: \ - --upgrade EOT } } From 76a095e81583072ea35697bb4fbde9c92af94e91 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 09:48:15 +0000 Subject: [PATCH 12/16] add github workflow --- .github/workflows/deploy_terraform.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 366cd004..3a234873 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -322,3 +322,27 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} + + # ============================================================ + # Deploy FastAPI Lambda + # ============================================================ + fast_api_lambda: + needs: [determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_fast_api + lambda_path: infrastructure/terraform/lambda/fast-api + stage: ${{ needs.determine_stage.outputs.stage }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} From 8828e4da15c6fe5329d7e9764e1d9b294ce55d97 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 10:04:02 +0000 Subject: [PATCH 13/16] move engine and fastapi depoyments to top of deploy_terraform --- .github/workflows/deploy_terraform.yml | 130 +++++++++++++------------ 1 file changed, 66 insertions(+), 64 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 3a234873..6291dd2a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -77,6 +77,71 @@ jobs: if: env.TERRAFORM_APPLY == 'true' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan + + # ============================================================ + # Ara Engine image and Push + # ============================================================ + ara_engine_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Ara Engine Lambda + # ============================================================ + ara_engine_lambda: + needs: [ara_engine_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_engine + lambda_path: infrastructure/terraform/lambda/engine + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + + # ============================================================ + # Deploy FastAPI Lambda + # ============================================================ + fast_api_lambda: + needs: [determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_fast_api + lambda_path: infrastructure/terraform/lambda/fast-api + stage: ${{ needs.determine_stage.outputs.stage }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} # ============================================================ # Build Address 2 UPRN image and Push @@ -241,47 +306,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - # ============================================================ - # Ara Engine image and Push - # ============================================================ - ara_engine_image: - needs: [determine_stage, shared_terraform] - uses: ./.github/workflows/_build_image.yml - with: - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/docker/engine.Dockerfile - build_context: . - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - - # ============================================================ - # Deploy Ara Engine Lambda - # ============================================================ - ara_engine_lambda: - needs: [ara_engine_image, determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: ara_engine - lambda_path: infrastructure/terraform/lambda/engine - stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} - terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} - TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} - TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} - TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} - TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} - TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} - TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} - TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} - # ============================================================ # Build OrdanceSurvey image and Push # ============================================================ @@ -323,26 +347,4 @@ jobs: AWS_REGION: ${{ secrets.DEV_AWS_REGION }} TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} - # ============================================================ - # Deploy FastAPI Lambda - # ============================================================ - fast_api_lambda: - needs: [determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: ara_fast_api - lambda_path: infrastructure/terraform/lambda/fast-api - stage: ${{ needs.determine_stage.outputs.stage }} - terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} - TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} - TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} - TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} - TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} - TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} - TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} - TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + From 7cade00db0658582542ccd018d2fec683bfd60e2 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 10:54:44 +0000 Subject: [PATCH 14/16] commented out custom domain stuff for now --- .../terraform/lambda/fast-api/main.tf | 19 ++++--- .../modules/lambda_with_api_gateway/main.tf | 52 +++++++++---------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index c87897c7..d9377b79 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -39,14 +39,13 @@ locals { db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) } +# data "aws_ssm_parameter" "certificate_arn" { +# name = "/ssl_certificate_arn" +# } -data "aws_ssm_parameter" "certificate_arn" { - name = "/ssl_certificate_arn" -} - -data "aws_route53_zone" "this" { - name = var.domain_name -} +# data "aws_route53_zone" "this" { +# name = var.domain_name +# } ############################################ # Install Python requirements @@ -84,9 +83,9 @@ module "fastapi" { timeout = 600 memory_size = 512 - domain_name = "api.${var.domain_name}" - certificate_arn = data.aws_ssm_parameter.certificate_arn.value - route53_zone_id = data.aws_route53_zone.this.zone_id + # domain_name = "api.${var.domain_name}" + # certificate_arn = data.aws_ssm_parameter.certificate_arn.value + # route53_zone_id = data.aws_route53_zone.this.zone_id environment = { ENVIRONMENT = var.stage diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf index 23ccc4b1..61e24c32 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf @@ -71,33 +71,33 @@ resource "aws_lambda_permission" "apigw_invoke" { ############################################ # Custom domain ############################################ -resource "aws_apigatewayv2_domain_name" "this" { - count = var.domain_name != null ? 1 : 0 - domain_name = var.domain_name +# resource "aws_apigatewayv2_domain_name" "this" { +# count = var.domain_name != null ? 1 : 0 +# domain_name = var.domain_name - domain_name_configuration { - certificate_arn = var.certificate_arn - endpoint_type = "REGIONAL" - security_policy = "TLS_1_2" - } -} +# domain_name_configuration { +# certificate_arn = var.certificate_arn +# endpoint_type = "REGIONAL" +# security_policy = "TLS_1_2" +# } +# } -resource "aws_apigatewayv2_api_mapping" "this" { - count = var.domain_name != null ? 1 : 0 - api_id = aws_apigatewayv2_api.this.id - domain_name = aws_apigatewayv2_domain_name.this[0].id - stage = aws_apigatewayv2_stage.this.id -} +# resource "aws_apigatewayv2_api_mapping" "this" { +# count = var.domain_name != null ? 1 : 0 +# api_id = aws_apigatewayv2_api.this.id +# domain_name = aws_apigatewayv2_domain_name.this[0].id +# stage = aws_apigatewayv2_stage.this.id +# } -resource "aws_route53_record" "this" { - count = var.domain_name != null ? 1 : 0 - name = aws_apigatewayv2_domain_name.this[0].domain_name - type = "A" - zone_id = var.route53_zone_id +# resource "aws_route53_record" "this" { +# count = var.domain_name != null ? 1 : 0 +# name = aws_apigatewayv2_domain_name.this[0].domain_name +# type = "A" +# zone_id = var.route53_zone_id - alias { - name = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].target_domain_name - zone_id = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].hosted_zone_id - evaluate_target_health = false - } -} \ No newline at end of file +# alias { +# name = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].target_domain_name +# zone_id = aws_apigatewayv2_domain_name.this[0].domain_name_configuration[0].hosted_zone_id +# evaluate_target_health = false +# } +# } \ No newline at end of file From 05843ab46263f1a150830e5c017a0cf3b77bb6b0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 10:56:09 +0000 Subject: [PATCH 15/16] comment out custom_domain_endpoint in outputs.tf --- .../terraform/modules/lambda_with_api_gateway/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf index 52db1ff9..9ced7c8b 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf @@ -6,6 +6,6 @@ output "api_endpoint" { value = aws_apigatewayv2_stage.this.invoke_url } -output "custom_domain_endpoint" { - value = var.domain_name != null ? "https://${var.domain_name}" : null -} \ No newline at end of file +# output "custom_domain_endpoint" { +# value = var.domain_name != null ? "https://${var.domain_name}" : null +# } \ No newline at end of file From 1a161396805d192cee3241fa11fac6098a6e5011 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 11:39:24 +0000 Subject: [PATCH 16/16] comment out DOMAIN_NAME --- infrastructure/terraform/lambda/fast-api/main.tf | 2 +- infrastructure/terraform/lambda/fast-api/variables.tf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index d9377b79..0a40b14c 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -91,7 +91,7 @@ module "fastapi" { ENVIRONMENT = var.stage API_KEY = var.api_key SECRET_KEY = var.secret_key - DOMAIN_NAME = var.domain_name + # DOMAIN_NAME = var.domain_name EPC_AUTH_TOKEN = var.epc_auth_token GOOGLE_SOLAR_API_KEY = var.google_solar_api_key diff --git a/infrastructure/terraform/lambda/fast-api/variables.tf b/infrastructure/terraform/lambda/fast-api/variables.tf index a3157590..d329e0ca 100644 --- a/infrastructure/terraform/lambda/fast-api/variables.tf +++ b/infrastructure/terraform/lambda/fast-api/variables.tf @@ -29,9 +29,9 @@ variable "secret_key" { sensitive = true } -variable "domain_name" { - type = string -} +# variable "domain_name" { +# type = string +# } variable "epc_auth_token" { type = string