From 8cf2d9d95a56854d2d349a9fea195d6b74a838bc Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 14:49:09 +0000 Subject: [PATCH 1/8] pull infrastructure changes from Jun-te's branch --- .../terraform/lambda/_template/main.tf | 49 +++++++++++ .../terraform/lambda/address2UPRN/main.tf | 49 +++++++++-- .../terraform/lambda/address2UPRN/outputs.tf | 14 ++++ .../terraform/lambda/condition-etl/main.tf | 1 - .../lambda/modules/lambda_with_sqs/outputs.tf | 1 + .../terraform/lambda/postcodeSplitter/main.tf | 83 ++++++++++++++++++- .../lambda/postcodeSplitter/variables.tf | 9 ++ .../modules/general_iam_policy/main.tf | 21 +++++ .../modules/general_iam_policy/outputs.tf | 9 ++ .../modules/general_iam_policy/variables.tf | 32 +++++++ .../modules/lambda_execution_role/main.tf | 16 ---- .../terraform/modules/s3_iam_policy/main.tf | 31 +++++++ .../modules/s3_iam_policy/outputs.tf | 14 ++++ .../modules/s3_iam_policy/variables.tf | 42 ++++++++++ infrastructure/terraform/shared/main.tf | 77 +++++++++++------ 15 files changed, 398 insertions(+), 50 deletions(-) create mode 100644 infrastructure/terraform/lambda/address2UPRN/outputs.tf create mode 100644 infrastructure/terraform/modules/general_iam_policy/main.tf create mode 100644 infrastructure/terraform/modules/general_iam_policy/outputs.tf create mode 100644 infrastructure/terraform/modules/general_iam_policy/variables.tf create mode 100644 infrastructure/terraform/modules/s3_iam_policy/main.tf create mode 100644 infrastructure/terraform/modules/s3_iam_policy/outputs.tf create mode 100644 infrastructure/terraform/modules/s3_iam_policy/variables.tf diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 3010aa8a..7f60d684 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -1,3 +1,30 @@ +# ============================================================================== +# TEMPLATE: Lambda Configuration with Optional S3 IAM Policy +# ============================================================================== +# Instructions: +# 1. Replace "REPLACE ME" with your lambda name (e.g., "my-lambda-name") +# 2. Add any additional environment variables as needed +# 3. To attach S3 IAM policies from shared state: +# - Uncomment the S3 policy attachment section below +# - Update the policy_arn to match the output from shared/main.tf +# - Available shared outputs (examples): +# - data.terraform_remote_state.shared.outputs.condition_etl_s3_read_arn +# - data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn +# 4. To create a NEW S3 policy: +# - Add a new module "lambda_s3_policy" in shared/main.tf using the +# s3_iam_policy module (see examples in shared/main.tf) +# - Then reference it here using data.terraform_remote_state.shared.outputs +# ============================================================================== + +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + module "lambda" { source = "../modules/lambda_with_sqs" @@ -12,3 +39,25 @@ module "lambda" { LOG_LEVEL = "info" } } + +# ====================================================================== +# OPTIONAL: Attach S3 IAM policy to Lambda execution role +# ====================================================================== +# Uncomment and configure the resource below to attach S3 permissions +# +# Example 1: Attach existing policy from shared state +# resource "aws_iam_role_policy_attachment" "lambda_s3_policy" { +# role = module.lambda.role_name +# policy_arn = data.terraform_remote_state.shared.outputs.YOUR_POLICY_OUTPUT_NAME_arn +# } +# +# Example 2: Attach multiple policies +# resource "aws_iam_role_policy_attachment" "lambda_read_policy" { +# role = module.lambda.role_name +# policy_arn = data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn +# } +# +# resource "aws_iam_role_policy_attachment" "lambda_write_policy" { +# role = module.lambda.role_name +# policy_arn = data.terraform_remote_state.shared.outputs.another_policy_arn +# } diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index 46b193f2..5a36153e 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -1,3 +1,19 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + module "address2uprn" { source = "../modules/lambda_with_sqs" @@ -6,9 +22,32 @@ module "address2uprn" { image_uri = local.image_uri - - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - } + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + GOOGLE_SOLAR_API_KEY = "test" + SAP_PREDICTIONS_BUCKET = "test" + CARBON_PREDICTIONS_BUCKET = "test" + HEAT_PREDICTIONS_BUCKET = "test" + HEATING_KWH_PREDICTIONS_BUCKET = "test" + HOTWATER_KWH_PREDICTIONS_BUCKET = "test" + API_KEY = "test" + ENVIRONMENT = "test" + SECRET_KEY = "test" + PLAN_TRIGGER_BUCKET = "test" + DATA_BUCKET = "test" + ENGINE_SQS_URL = "test" + ENERGY_ASSESSMENTS_BUCKET = "test" + S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name + }, + ) } + +# Attach S3 read policy to the Lambda execution role +resource "aws_iam_role_policy_attachment" "address2uprn_read_and_write" { + role = module.address2uprn.role_name + policy_arn = data.terraform_remote_state.shared.outputs.address_2_uprn_s3_read_and_write_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/outputs.tf b/infrastructure/terraform/lambda/address2UPRN/outputs.tf new file mode 100644 index 00000000..e4645a0a --- /dev/null +++ b/infrastructure/terraform/lambda/address2UPRN/outputs.tf @@ -0,0 +1,14 @@ +output "address2uprn_queue_url" { + value = module.address2uprn.queue_url + description = "URL of the address2UPRN SQS queue" +} + +output "address2uprn_queue_arn" { + value = module.address2uprn.queue_arn + description = "ARN of the address2UPRN SQS queue" +} + +output "address2uprn_lambda_arn" { + value = module.address2uprn.lambda_arn + description = "ARN of the address2UPRN Lambda function" +} diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 4219f209..0128f975 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -23,7 +23,6 @@ module "lambda" { stage = var.stage image_uri = local.image_uri - timeout = 180 environment = merge( diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf b/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf index afc9246d..b408593f 100644 --- a/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf +++ b/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf @@ -9,3 +9,4 @@ output "queue_arn" { output "queue_url" { value = module.queue.queue_url } + diff --git a/infrastructure/terraform/lambda/postcodeSplitter/main.tf b/infrastructure/terraform/lambda/postcodeSplitter/main.tf index ebbdbfdc..d37a01c9 100644 --- a/infrastructure/terraform/lambda/postcodeSplitter/main.tf +++ b/infrastructure/terraform/lambda/postcodeSplitter/main.tf @@ -1,3 +1,30 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +# Reference the existing address2UPRN Lambda outputs from address2uprn state +data "terraform_remote_state" "address2uprn" { + backend = "s3" + config = { + bucket = "address2uprn-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + module "lambda" { source = "../modules/lambda_with_sqs" @@ -7,8 +34,56 @@ module "lambda" { image_uri = local.image_uri - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - } + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + GOOGLE_SOLAR_API_KEY = "test" + SAP_PREDICTIONS_BUCKET = "test" + CARBON_PREDICTIONS_BUCKET = "test" + HEAT_PREDICTIONS_BUCKET = "test" + HEATING_KWH_PREDICTIONS_BUCKET = "test" + HOTWATER_KWH_PREDICTIONS_BUCKET = "test" + API_KEY = "test" + ENVIRONMENT = "test" + SECRET_KEY = "test" + PLAN_TRIGGER_BUCKET = "test" + DATA_BUCKET = "test" + EPC_AUTH_TOKEN = "test" + ENGINE_SQS_URL = "test" + ENERGY_ASSESSMENTS_BUCKET = "test" + ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url + S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name + }, + ) } + +# Attach S3 read policy to the Lambda execution role +resource "aws_iam_role_policy_attachment" "postcode_splitter_s3_read" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn +} + +# Create SQS send policy for address2UPRN queue +module "postcode_splitter_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "postcode-splitter-sqs-send-${var.stage}" + policy_description = "Allow postcode-splitter Lambda to send messages to address2UPRN queue" + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_arn + ] +} + +# Attach SQS policy to the Lambda execution role +resource "aws_iam_role_policy_attachment" "postcode_splitter_sqs_send" { + role = module.lambda.role_name + policy_arn = module.postcode_splitter_sqs_policy.policy_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/postcodeSplitter/variables.tf b/infrastructure/terraform/lambda/postcodeSplitter/variables.tf index 9ce45fa5..7bd68543 100644 --- a/infrastructure/terraform/lambda/postcodeSplitter/variables.tf +++ b/infrastructure/terraform/lambda/postcodeSplitter/variables.tf @@ -24,3 +24,12 @@ locals { output "resolved_image_uri" { value = local.image_uri } + + + + + + + + + diff --git a/infrastructure/terraform/modules/general_iam_policy/main.tf b/infrastructure/terraform/modules/general_iam_policy/main.tf new file mode 100644 index 00000000..f7ffe4a1 --- /dev/null +++ b/infrastructure/terraform/modules/general_iam_policy/main.tf @@ -0,0 +1,21 @@ +# IAM Policy with dynamic actions and resources +resource "aws_iam_policy" "policy" { + name = var.policy_name + description = var.policy_description + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + merge( + { + Effect = "Allow" + Action = var.actions + Resource = var.resources + }, + var.conditions != null ? { Condition = var.conditions } : {} + ) + ] + }) + + tags = var.tags +} diff --git a/infrastructure/terraform/modules/general_iam_policy/outputs.tf b/infrastructure/terraform/modules/general_iam_policy/outputs.tf new file mode 100644 index 00000000..cfceab05 --- /dev/null +++ b/infrastructure/terraform/modules/general_iam_policy/outputs.tf @@ -0,0 +1,9 @@ +output "policy_arn" { + value = aws_iam_policy.policy.arn + description = "ARN of the created IAM policy" +} + +output "policy_name" { + value = aws_iam_policy.policy.name + description = "Name of the created IAM policy" +} diff --git a/infrastructure/terraform/modules/general_iam_policy/variables.tf b/infrastructure/terraform/modules/general_iam_policy/variables.tf new file mode 100644 index 00000000..0d824eb5 --- /dev/null +++ b/infrastructure/terraform/modules/general_iam_policy/variables.tf @@ -0,0 +1,32 @@ +variable "policy_name" { + description = "Name of the IAM policy" + type = string +} + +variable "policy_description" { + description = "Description of the IAM policy" + type = string + default = "" +} + +variable "actions" { + description = "List of IAM actions allowed by this policy" + type = list(string) +} + +variable "resources" { + description = "List of AWS resources this policy applies to" + type = list(string) +} + +variable "conditions" { + description = "Optional IAM policy conditions" + type = any + default = null +} + +variable "tags" { + description = "Tags to apply to the policy" + type = map(string) + default = {} +} diff --git a/infrastructure/terraform/modules/lambda_execution_role/main.tf b/infrastructure/terraform/modules/lambda_execution_role/main.tf index fa657afd..e593b17c 100644 --- a/infrastructure/terraform/modules/lambda_execution_role/main.tf +++ b/infrastructure/terraform/modules/lambda_execution_role/main.tf @@ -19,19 +19,3 @@ resource "aws_iam_role_policy_attachment" "basic_logs" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" } -resource "aws_iam_role_policy" "ecr_pull" { - role = aws_iam_role.this.name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Action = [ - "ecr:GetAuthorizationToken", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer" - ] - Resource = "*" - }] - }) -} diff --git a/infrastructure/terraform/modules/s3_iam_policy/main.tf b/infrastructure/terraform/modules/s3_iam_policy/main.tf new file mode 100644 index 00000000..397bd963 --- /dev/null +++ b/infrastructure/terraform/modules/s3_iam_policy/main.tf @@ -0,0 +1,31 @@ +# Dynamically build S3 resources list from bucket ARNs and resource paths +locals { + # Generate full resource ARNs by combining bucket ARNs with resource paths + resources = flatten([ + for bucket_arn in var.bucket_arns : [ + for path in var.resource_paths : "${bucket_arn}${path}" + ] + ]) +} + +# IAM Policy with dynamic actions and resources +resource "aws_iam_policy" "s3_policy" { + name = var.policy_name + description = var.policy_description + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + merge( + { + Effect = "Allow" + Action = var.actions + Resource = local.resources + }, + var.conditions != null ? { Condition = var.conditions } : {} + ) + ] + }) + + tags = var.tags +} diff --git a/infrastructure/terraform/modules/s3_iam_policy/outputs.tf b/infrastructure/terraform/modules/s3_iam_policy/outputs.tf new file mode 100644 index 00000000..85defd9c --- /dev/null +++ b/infrastructure/terraform/modules/s3_iam_policy/outputs.tf @@ -0,0 +1,14 @@ +output "policy_arn" { + description = "ARN of the S3 IAM policy" + value = aws_iam_policy.s3_policy.arn +} + +output "policy_name" { + description = "Name of the S3 IAM policy" + value = aws_iam_policy.s3_policy.name +} + +output "policy_id" { + description = "ID of the S3 IAM policy" + value = aws_iam_policy.s3_policy.id +} diff --git a/infrastructure/terraform/modules/s3_iam_policy/variables.tf b/infrastructure/terraform/modules/s3_iam_policy/variables.tf new file mode 100644 index 00000000..e2b3d7a8 --- /dev/null +++ b/infrastructure/terraform/modules/s3_iam_policy/variables.tf @@ -0,0 +1,42 @@ +variable "policy_name" { + description = "Name of the IAM policy" + type = string +} + +variable "policy_description" { + description = "Description of the IAM policy" + type = string + default = "" +} + +variable "bucket_arns" { + description = "List of S3 bucket ARNs to grant access to" + type = list(string) +} + +variable "actions" { + description = "List of S3 actions to allow (e.g., ['s3:GetObject'], ['s3:PutObject'], ['s3:DeleteObject'])" + type = list(string) + default = ["s3:GetObject"] +} + +variable "resource_paths" { + description = "List of resource paths within buckets (e.g., ['/*'] for all objects, ['/specific-prefix/*'] for specific prefix)" + type = list(string) + default = ["/*"] +} + +variable "conditions" { + description = "Optional IAM policy conditions to apply to the statement" + type = any + default = null +} + +variable "tags" { + description = "Tags to apply to the policy" + type = map(string) + default = {} +} + + + diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index b1474055..acf8c281 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -133,6 +133,11 @@ module "retrofit_sap_data" { allowed_origins = var.allowed_origins } +output "retrofit_sap_data_bucket_name" { + value = module.retrofit_sap_data.bucket_name + description = "Name of the retrofit SAP data bucket" +} + module "retrofit_carbon_predictions" { source = "../modules/s3" bucketname = "retrofit-carbon-predictions-${var.stage}" @@ -305,6 +310,21 @@ module "address2uprn_registry" { } +# S3 policy for postcode splitter to read from retrofit data bucket +module "address2uprn_s3_read_and_write" { + source = "../modules/s3_iam_policy" + + policy_name = "Address2UPRNReadandWriteS3" + policy_description = "Allow address2uprn Lambda to read and write from retrofit-data bucket" + bucket_arns = ["arn:aws:s3:::retrofit-data-${var.stage}"] + actions = ["s3:GetObject", "s3:ListBucket", "s3:PutObject"] + resource_paths = ["/*"] +} + +output "address_2_uprn_s3_read_and_write_arn" { + value = module.address2uprn_s3_read_and_write.policy_arn +} + ################################################ # Condition ETL – Lambda ECR ################################################ @@ -321,6 +341,28 @@ module "condition_etl_registry" { } +# Condition Data S3 Bucket to store initial data +module "condition_data_bucket" { + source = "../modules/s3" + bucketname = "condition-data-${var.stage}" + allowed_origins = var.allowed_origins +} + +module "condition_etl_s3_read" { + source = "../modules/s3_iam_policy" + + policy_name = "ConditionETLReadS3" + policy_description = "Allow Lambda to read objects from condition-data-${var.stage}" + bucket_arns = ["arn:aws:s3:::condition-data-${var.stage}"] + actions = ["s3:GetObject"] + resource_paths = ["/*"] +} + +output "condition_etl_s3_read_arn" { + value = module.condition_etl_s3_read.policy_arn +} + + ################################################ # Postcode Splitter – Lambda ECR ################################################ @@ -337,30 +379,17 @@ module "postcode_splitter_registry" { } -################################################ -# Conidition data – S3 bucket -################################################ -module "condition_data_bucket" { - source = "../modules/s3" - bucketname = "condition-data-${var.stage}" - allowed_origins = var.allowed_origins +# S3 policy for postcode splitter to read from retrofit data bucket +module "postcode_splitter_s3_read" { + source = "../modules/s3_iam_policy" + + policy_name = "PostcodeSplitterReadS3" + policy_description = "Allow postcode splitter Lambda to read from retrofit-data bucket" + bucket_arns = ["arn:aws:s3:::retrofit-data-${var.stage}"] + actions = ["s3:GetObject", "s3:ListBucket", "s3:PutObject"] + resource_paths = ["/*"] } -resource "aws_iam_policy" "condition_etl_s3_read" { - name = "ConditionETLReadS3" - description = "Allow Lambda to read objects from condition-data-${var.stage}" - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["s3:GetObject"] - Resource = "arn:aws:s3:::condition-data-${var.stage}/*" - } - ] - }) -} - -output "condition_etl_s3_read_arn" { - value = aws_iam_policy.condition_etl_s3_read.arn +output "postcode_splitter_s3_read_arn" { + value = module.postcode_splitter_s3_read.policy_arn } \ No newline at end of file From 7246c953455b8a4d78cf02ed1b6b5380a93af24c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 15:05:42 +0000 Subject: [PATCH 2/8] categorisation terraform --- .github/workflows/deploy_terraform.yml | 40 +++++++++++++++++++ .../terraform/lambda/categorisation/main.tf | 27 +++++++++++++ .../lambda/categorisation/provider.tf | 16 ++++++++ .../lambda/categorisation/variables.tf | 27 +++++++++++++ infrastructure/terraform/shared/main.tf | 16 ++++++++ 5 files changed, 126 insertions(+) create mode 100644 infrastructure/terraform/lambda/categorisation/main.tf create mode 100644 infrastructure/terraform/lambda/categorisation/provider.tf create mode 100644 infrastructure/terraform/lambda/categorisation/variables.tf diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 71e2ad9d..fca44a4c 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -189,6 +189,46 @@ jobs: ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Categorisation image and Push + # ============================================================ + categorisation_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/categorisation/handler/Dockerfile + build_context: . + build_args: | + DEV_DB_HOST=$DEV_DB_HOST + DEV_DB_PORT=$DEV_DB_PORT + DEV_DB_NAME=$DEV_DB_NAME + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} + DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} + DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} + + # ============================================================ + # Deploy Categorisation Lambda + # ============================================================ + condition_etl_lambda: + needs: [categorisation_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: categorisation + lambda_path: infrastructure/terraform/lambda/categorisation + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.categorisation_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/infrastructure/terraform/lambda/categorisation/main.tf b/infrastructure/terraform/lambda/categorisation/main.tf new file mode 100644 index 00000000..a402a386 --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/main.tf @@ -0,0 +1,27 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +module "lambda" { + source = "../modules/lambda_with_sqs" + + name = "categorisation" + stage = var.stage + + image_uri = local.image_uri + + + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + } + ) +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/categorisation/provider.tf b/infrastructure/terraform/lambda/categorisation/provider.tf new file mode 100644 index 00000000..37c412ce --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + backend "s3" { + bucket = REPLACE_ME + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/categorisation/variables.tf b/infrastructure/terraform/lambda/categorisation/variables.tf new file mode 100644 index 00000000..e4bab243 --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/variables.tf @@ -0,0 +1,27 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index acf8c281..2e009196 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -392,4 +392,20 @@ module "postcode_splitter_s3_read" { output "postcode_splitter_s3_read_arn" { value = module.postcode_splitter_s3_read.policy_arn +} + +################################################ +# Categorisation – Lambda ECR +################################################ +module "categorisation_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "categorisation-terraform-state" + +} + +module "categorisation_registry" { + source = "../modules/container_registry" + name = "categorisation" + stage = var.stage + } \ No newline at end of file From 36c710f6e65b9b0b01c3e7451e9d409e80416bf4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 15:06:36 +0000 Subject: [PATCH 3/8] correction to categorisation terraform --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index fca44a4c..216069fe 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -219,7 +219,7 @@ jobs: # ============================================================ # Deploy Categorisation Lambda # ============================================================ - condition_etl_lambda: + categorisation_lambda: needs: [categorisation_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: From 9dd6f4f72ea33d4ac4bebd3adc39e8cbb6de1e41 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 15:29:08 +0000 Subject: [PATCH 4/8] remove categorisation stuff from deploy_terraform --- .github/workflows/deploy_terraform.yml | 40 ------------------- .../terraform/lambda/_template/README.md | 4 +- 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 216069fe..71e2ad9d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -189,46 +189,6 @@ jobs: ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - - # ============================================================ - # Categorisation image and Push - # ============================================================ - categorisation_image: - needs: [determine_stage, shared_terraform] - uses: ./.github/workflows/_build_image.yml - with: - ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/categorisation/handler/Dockerfile - build_context: . - build_args: | - DEV_DB_HOST=$DEV_DB_HOST - DEV_DB_PORT=$DEV_DB_PORT - DEV_DB_NAME=$DEV_DB_NAME - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} - DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} - DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} - - # ============================================================ - # Deploy Categorisation Lambda - # ============================================================ - categorisation_lambda: - needs: [categorisation_image, determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: categorisation - lambda_path: infrastructure/terraform/lambda/categorisation - stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.categorisation_image.outputs.image_digest }} - terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index a7282fc9..7992ec5c 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -3,7 +3,7 @@ ### 1. Create the Lambda scaffold - Copy the template: - cp -r lambda/_template lambda/ + `cp -r lambda/_template lambda/` --- @@ -13,7 +13,7 @@ infrastructure/terraform/shared/main.tf - Apply the shared stack - - This requires commenting 'if env.stage == "prod"' in .github/workflows/deploy_terraform.yml + - This requires commenting `if: env.STAGE == 'prod'` in .github/workflows/deploy_terraform.yml - Verify the ECR repository exists in AWS From 42cac343576a4cf1f0bb2c02df145dd8e53ed293 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 16 Feb 2026 15:50:01 +0000 Subject: [PATCH 5/8] only run on branches it was told to --- .github/workflows/deploy_terraform.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index e7c8fb94..6280abcd 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -77,10 +77,10 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply - if: env.STAGE == 'prod' + if: env.TERRAFORM_APPLY == 'true' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan - + # ============================================================ # 2️⃣ Build Address 2 UPRN image and Push # ============================================================ From 2fc01a5dc7f059c821126962796d519b23684045 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 15:59:42 +0000 Subject: [PATCH 6/8] replace replace me in provider.tf --- infrastructure/terraform/lambda/categorisation/provider.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/categorisation/provider.tf b/infrastructure/terraform/lambda/categorisation/provider.tf index 37c412ce..f983533d 100644 --- a/infrastructure/terraform/lambda/categorisation/provider.tf +++ b/infrastructure/terraform/lambda/categorisation/provider.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - bucket = REPLACE_ME + bucket = "categorisation" key = "terraform.tfstate" region = "eu-west-2" } From c96fa51badc00dfb6df770fa78c82006f37d63c4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 16:09:49 +0000 Subject: [PATCH 7/8] add terraform_apply env var to shared_terraform --- .github/workflows/deploy_terraform.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 6280abcd..691c9996 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -52,6 +52,7 @@ jobs: runs-on: ubuntu-latest env: STAGE: ${{ needs.determine_stage.outputs.stage }} + TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }} steps: - uses: actions/checkout@v4 From 146ca3d8b0d1ebdd577c15663da0a4d7bc3b4a2d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 16 Feb 2026 16:12:36 +0000 Subject: [PATCH 8/8] update readme --- infrastructure/terraform/lambda/_template/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 7992ec5c..5bb10627 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -12,8 +12,7 @@ infrastructure/terraform/shared/main.tf -- Apply the shared stack - - This requires commenting `if: env.STAGE == 'prod'` in .github/workflows/deploy_terraform.yml +- Create a PR to deploy this to main then dev in order to deploy the shared stack - Verify the ECR repository exists in AWS