From c7c208334e231568449acb714a42b12d0e47405d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 16 Feb 2026 16:54:35 +0000 Subject: [PATCH 1/7] lets view plan for now --- .github/workflows/deploy_terraform.yml | 1 + infrastructure/terraform/lambda/address2UPRN/main.tf | 2 ++ infrastructure/terraform/modules/sqs_queue/main.tf | 2 +- infrastructure/terraform/modules/sqs_queue/variables.tf | 7 ++++++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index e7c8fb94..c7a3ec1b 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -52,6 +52,7 @@ jobs: runs-on: ubuntu-latest env: STAGE: ${{ needs.determine_stage.outputs.stage }} + TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }} steps: - uses: actions/checkout@v4 diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index 5a36153e..f53d55c8 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -22,6 +22,8 @@ module "address2uprn" { image_uri = local.image_uri + timeout = 900 + environment = merge( { STAGE = var.stage diff --git a/infrastructure/terraform/modules/sqs_queue/main.tf b/infrastructure/terraform/modules/sqs_queue/main.tf index 580e67bd..356d3429 100644 --- a/infrastructure/terraform/modules/sqs_queue/main.tf +++ b/infrastructure/terraform/modules/sqs_queue/main.tf @@ -5,7 +5,7 @@ resource "aws_sqs_queue" "dlq" { resource "aws_sqs_queue" "this" { name = var.name - visibility_timeout_seconds = 120 + visibility_timeout_seconds = var.timeout * 6 redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.dlq.arn diff --git a/infrastructure/terraform/modules/sqs_queue/variables.tf b/infrastructure/terraform/modules/sqs_queue/variables.tf index 943a7a16..bb8e4485 100644 --- a/infrastructure/terraform/modules/sqs_queue/variables.tf +++ b/infrastructure/terraform/modules/sqs_queue/variables.tf @@ -2,5 +2,10 @@ variable "name" { type = string } variable "max_receive_count" { type = number - default = 5 + default = 1 +} + +variable "timeout" { + type = number + default = 900 } From 952a2cb2c0df65bf4c58c62e68c5a2d7d7a516f3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 16 Feb 2026 17:23:15 +0000 Subject: [PATCH 2/7] quick shared infra --- .github/workflows/deploy_terraform.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index c7a3ec1b..c6937f7a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -3,7 +3,9 @@ name: Deploy infrastructure on: push: branches: - - "**" + - "main" + - "dev" + - "prod" paths: - 'infrastructure/terraform/**' - '.github/workflows/deploy_terraform.yml' From 0afa8b3aba6993d45a35336ce0f878eb56e32faa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 16 Feb 2026 17:29:10 +0000 Subject: [PATCH 3/7] sqs time out redploy --- infrastructure/terraform/modules/sqs_queue/main.tf | 2 +- infrastructure/terraform/modules/sqs_queue/variables.tf | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/infrastructure/terraform/modules/sqs_queue/main.tf b/infrastructure/terraform/modules/sqs_queue/main.tf index 356d3429..afb7dc27 100644 --- a/infrastructure/terraform/modules/sqs_queue/main.tf +++ b/infrastructure/terraform/modules/sqs_queue/main.tf @@ -5,7 +5,7 @@ resource "aws_sqs_queue" "dlq" { resource "aws_sqs_queue" "this" { name = var.name - visibility_timeout_seconds = var.timeout * 6 + visibility_timeout_seconds = 1000 redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.dlq.arn diff --git a/infrastructure/terraform/modules/sqs_queue/variables.tf b/infrastructure/terraform/modules/sqs_queue/variables.tf index bb8e4485..95b33231 100644 --- a/infrastructure/terraform/modules/sqs_queue/variables.tf +++ b/infrastructure/terraform/modules/sqs_queue/variables.tf @@ -4,8 +4,3 @@ variable "max_receive_count" { type = number default = 1 } - -variable "timeout" { - type = number - default = 900 -} From 9d87ef8eec2b0b55637486b208da4710ffe18d27 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 17 Feb 2026 09:17:41 +0000 Subject: [PATCH 4/7] deploy categorisation image and lambda --- .github/workflows/deploy_terraform.yml | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 691c9996..f2a1f34a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -205,3 +205,44 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # Categorisation image and Push + # ============================================================ + categorisation_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/categorisation/handler/Dockerfile + build_context: . + build_args: | + DEV_DB_HOST=$DEV_DB_HOST + DEV_DB_PORT=$DEV_DB_PORT + DEV_DB_NAME=$DEV_DB_NAME + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} + DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} + DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} + + # ============================================================ + # Deploy Categorisation Lambda + # ============================================================ + categorisation_lambda: + needs: [categorisation_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: categorisation + lambda_path: infrastructure/terraform/lambda/categorisation + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.categorisation_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + From 1d6746d3911b2b3ff1c5d8eb713a8891208a6b7d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 17 Feb 2026 09:26:04 +0000 Subject: [PATCH 5/7] correct terraform state bucket --- infrastructure/terraform/lambda/categorisation/provider.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/categorisation/provider.tf b/infrastructure/terraform/lambda/categorisation/provider.tf index f983533d..fe497c81 100644 --- a/infrastructure/terraform/lambda/categorisation/provider.tf +++ b/infrastructure/terraform/lambda/categorisation/provider.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - bucket = "categorisation" + bucket = "categorisation-terraform-state" key = "terraform.tfstate" region = "eu-west-2" } From 827745c10f0e8354ad522b7641919ce875e5fa9d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 17 Feb 2026 09:29:37 +0000 Subject: [PATCH 6/7] add db credentials to categorisation tf --- infrastructure/terraform/lambda/categorisation/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infrastructure/terraform/lambda/categorisation/main.tf b/infrastructure/terraform/lambda/categorisation/main.tf index a402a386..6e30dd8e 100644 --- a/infrastructure/terraform/lambda/categorisation/main.tf +++ b/infrastructure/terraform/lambda/categorisation/main.tf @@ -1,3 +1,7 @@ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + data "terraform_remote_state" "shared" { backend = "s3" config = { @@ -7,6 +11,10 @@ data "terraform_remote_state" "shared" { } } +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + module "lambda" { source = "../modules/lambda_with_sqs" From 35f7738160ce40e61c048facc4b22608ee643f6e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 17 Feb 2026 09:33:31 +0000 Subject: [PATCH 7/7] implement handler --- backend/categorisation/handler/handler.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/categorisation/handler/handler.py b/backend/categorisation/handler/handler.py index e74bfeb5..20076613 100644 --- a/backend/categorisation/handler/handler.py +++ b/backend/categorisation/handler/handler.py @@ -1,4 +1,9 @@ +import json from typing import Any, Mapping +from backend.categorisation.categorisation_trigger_request import ( + CategorisationTriggerRequest, +) +from backend.categorisation.processor import process_portfolio from utils.logger import setup_logger @@ -7,4 +12,15 @@ logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: - pass + for record in event.get("Records", []): + try: + body_dict = json.loads(record["body"]) + logger.debug("Validating request body") + payload = CategorisationTriggerRequest.model_validate(body_dict) + + logger.debug("Successfully validated request body") + + process_portfolio(payload.portfolio_id) + + except Exception as e: + logger.error(f"Failed to process record: {e}")