From b8283abc45c0752ba9abae85a23e78d1e81efbe4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 11:16:34 +0000 Subject: [PATCH 01/50] try getting all db vars from github secrets --- .github/workflows/_build_image.yml | 13 ++++-- .github/workflows/deploy_terraform.yml | 9 ++-- backend/.env.local | 64 +++++++++++++------------- backend/condition/handler/Dockerfile | 27 ++++------- backend/condition/handler/handler.py | 16 +++---- 5 files changed, 61 insertions(+), 68 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index fce856b6..41677b1f 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -34,13 +34,20 @@ on: required: true DEV_DB_HOST: required: false - REAL_DB_HOST: + DEV_DB_PORT: + required: false + DEV_DB_NAME: required: false jobs: build: runs-on: ubuntu-latest + env: + DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} + DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} + DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} + outputs: image_digest: ${{ steps.digest.outputs.image_digest }} ecr_repo_url: ${{ steps.repo.outputs.ecr_repo_url }} @@ -82,8 +89,8 @@ jobs: done <<< "${{ inputs.build_args }}" echo "dev db host: $DEV_DB_HOST" - echo "real db host: $REAL_DB_HOST" - echo "aws_key_id: $AWS_ACCESS_KEY_ID" + echo "dev db port: $DEV_DB_PORT" + echo "dev db name: $DEV_DB_NAME" docker build \ -f ${{ inputs.dockerfile_path }} \ diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index f8718119..4ac08e41 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -142,17 +142,16 @@ jobs: dockerfile_path: backend/condition/handler/Dockerfile build_context: . build_args: | - JUNTE=best DEV_DB_HOST=$DEV_DB_HOST - REAL_DB_HOST=$REAL_DB_HOST - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_REGION=$AWS_REGION + DEV_DB_PORT=$DEV_DB_PORT + DEV_DB_NAME=$DEV_DB_NAME secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} - REAL_DB_HOST: ${{ secrets.dev_DB_HOST }} + DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} + DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ # Deploy Condition ETL Lambda diff --git a/backend/.env.local b/backend/.env.local index 22e1db35..4e401ed9 100644 --- a/backend/.env.local +++ b/backend/.env.local @@ -6,35 +6,35 @@ DB_PASSWORD=makingwarmerhomes #not used -GOOGLE_SOLAR_API_KEY="test" -SAP_PREDICTIONS_BUCKET="test" -CARBON_PREDICTIONS_BUCKET="test" -HEAT_PREDICTIONS_BUCKET="test" -HEATING_KWH_PREDICTIONS_BUCKET="test" -HOTWATER_KWH_PREDICTIONS_BUCKET="test" -API_KEY="test" -ENVIRONMENT="test" -SECRET_KEY="test" -PLAN_TRIGGER_BUCKET="test" -DATA_BUCKET="test" -EPC_AUTH_TOKEN="test" -ENGINE_SQS_URL="test" -ENERGY_ASSESSMENTS_BUCKET="test" -API_KEY="test" -SECRET_KEY="test" -ENVIRONMENT="test" -DATA_BUCKET="test" -PLAN_TRIGGER_BUCKET="test" -ENGINE_SQS_URL="test" -GOOGLE_SOLAR_API_KEY="test" -DB_HOST="test" -DB_PASSWORD="test" -DB_USERNAME="test" -DB_PORT="test" -DB_NAME="test" -SAP_PREDICTIONS_BUCKET="test" -CARBON_PREDICTIONS_BUCKET="test" -HEAT_PREDICTIONS_BUCKET="test" -HEATING_KWH_PREDICTIONS_BUCKET="test" -HOTWATER_KWH_PREDICTIONS_BUCKET="test" -ENERGY_ASSESSMENTS_BUCKET="test" \ No newline at end of file +GOOGLE_SOLAR_API_KEY=test +SAP_PREDICTIONS_BUCKET=test +CARBON_PREDICTIONS_BUCKET=test +HEAT_PREDICTIONS_BUCKET=test +HEATING_KWH_PREDICTIONS_BUCKET=test +HOTWATER_KWH_PREDICTIONS_BUCKET=test +API_KEY=test +ENVIRONMENT=test +SECRET_KEY=test +PLAN_TRIGGER_BUCKET=test +DATA_BUCKET=test +EPC_AUTH_TOKEN=test +ENGINE_SQS_URL=test +ENERGY_ASSESSMENTS_BUCKET=test +API_KEY=test +SECRET_KEY=test +ENVIRONMENT=test +DATA_BUCKET=test +PLAN_TRIGGER_BUCKET=test +ENGINE_SQS_URL=test +GOOGLE_SOLAR_API_KEY=test +DB_HOST=test +DB_PASSWORD=test +DB_USERNAME=test +DB_PORT=test +DB_NAME=test +SAP_PREDICTIONS_BUCKET=test +CARBON_PREDICTIONS_BUCKET=test +HEAT_PREDICTIONS_BUCKET=test +HEATING_KWH_PREDICTIONS_BUCKET=test +HOTWATER_KWH_PREDICTIONS_BUCKET=test +ENERGY_ASSESSMENTS_BUCKET=test \ No newline at end of file diff --git a/backend/condition/handler/Dockerfile b/backend/condition/handler/Dockerfile index 5cb95532..031d981e 100644 --- a/backend/condition/handler/Dockerfile +++ b/backend/condition/handler/Dockerfile @@ -2,28 +2,21 @@ FROM public.ecr.aws/lambda/python:3.11 # For local running: # FROM python:3.11.10-bullseye -# ARG EPC_AUTH_TOKEN - - -# ARG DEV_DB_HOST -ARG JUNTE -ENV JUNTE=${JUNTE} - ARG DEV_DB_HOST -ENV DEV_DB_HOST=${DEV_DB_HOST} +ARG DEV_DB_PORT +ARG DEV_DB_NAME -ARG REAL_DB_HOST -ENV REAL_DB_HOST=${REAL_DB_HOST} - -ARG AWS_ACCESS_KEY_ID -ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - -ARG AWS_REGION -ENV AWS_REGION=${AWS_REGION} # Set working directory (Lambda task root) WORKDIR /var/task +# Environment +ENV DB_HOST=${DEV_DB_HOST} +ENV DB_PORT=${DEV_DB_PORT} +ENV DB_NAME=${DEV_DB_NAME} + +COPY backend/.env.local backend/.env.local + # ----------------------------- # Copy requirements FIRST (for Docker layer caching) # ----------------------------- @@ -47,8 +40,6 @@ COPY backend/app/__init__.py backend/app/__init__.py COPY backend/app/db/__init__.py backend/app/db/__init__.py -# ENV EPC_AUTH_TOKEN=${EPC_AUTH_TOKEN} - # ----------------------------- # Lambda handler # ----------------------------- diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 21fa6928..11e10997 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -16,20 +16,16 @@ import os def handler(event: Mapping[str, Any], context: Any) -> None: print( - "hello Jun-te", - os.getenv("JUNTE", "empty junte"), + "hello DB HOST:", + os.getenv("DB_HOST", "empty db host"), ) print( - "hello DEV DB HOST:", - os.getenv("DEV_DB_HOST", "empty db"), + "hello DB PORT:", + os.getenv("DB_PORT", "empty db port"), ) print( - "hello REAL DB HOST:", - os.getenv("REAL_DB_HOST", "empty db"), - ) - print( - "hello access key", - os.getenv("AWS_ACCESS_KEY_ID", "empty key"), + "hello DB NAME:", + os.getenv("DB_NAME", "empty name"), ) print( "hello region", From f10951f81df35661cbd28e9a8a9b537202bf550d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 11:24:00 +0000 Subject: [PATCH 02/50] try without build args --- .github/workflows/deploy_terraform.yml | 4 ---- backend/condition/handler/Dockerfile | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4ac08e41..def2a6dc 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -141,10 +141,6 @@ jobs: ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} dockerfile_path: backend/condition/handler/Dockerfile build_context: . - build_args: | - DEV_DB_HOST=$DEV_DB_HOST - DEV_DB_PORT=$DEV_DB_PORT - DEV_DB_NAME=$DEV_DB_NAME secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/backend/condition/handler/Dockerfile b/backend/condition/handler/Dockerfile index 031d981e..96035838 100644 --- a/backend/condition/handler/Dockerfile +++ b/backend/condition/handler/Dockerfile @@ -2,18 +2,18 @@ FROM public.ecr.aws/lambda/python:3.11 # For local running: # FROM python:3.11.10-bullseye -ARG DEV_DB_HOST -ARG DEV_DB_PORT -ARG DEV_DB_NAME +# ARG DEV_DB_HOST +# ARG DEV_DB_PORT +# ARG DEV_DB_NAME # Set working directory (Lambda task root) WORKDIR /var/task # Environment -ENV DB_HOST=${DEV_DB_HOST} -ENV DB_PORT=${DEV_DB_PORT} -ENV DB_NAME=${DEV_DB_NAME} +# ENV DB_HOST=${DEV_DB_HOST} +# ENV DB_PORT=${DEV_DB_PORT} +# ENV DB_NAME=${DEV_DB_NAME} COPY backend/.env.local backend/.env.local From 9ae81c1099d369805a17d273a6283d43a15aa354 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 11:39:53 +0000 Subject: [PATCH 03/50] reinstate build args and check whether imports work now --- .github/workflows/deploy_terraform.yml | 4 ++++ backend/condition/handler/Dockerfile | 12 ++++++------ backend/condition/handler/handler.py | 12 ++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index def2a6dc..4ac08e41 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -141,6 +141,10 @@ jobs: ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} dockerfile_path: backend/condition/handler/Dockerfile build_context: . + build_args: | + DEV_DB_HOST=$DEV_DB_HOST + DEV_DB_PORT=$DEV_DB_PORT + DEV_DB_NAME=$DEV_DB_NAME secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/backend/condition/handler/Dockerfile b/backend/condition/handler/Dockerfile index 96035838..031d981e 100644 --- a/backend/condition/handler/Dockerfile +++ b/backend/condition/handler/Dockerfile @@ -2,18 +2,18 @@ FROM public.ecr.aws/lambda/python:3.11 # For local running: # FROM python:3.11.10-bullseye -# ARG DEV_DB_HOST -# ARG DEV_DB_PORT -# ARG DEV_DB_NAME +ARG DEV_DB_HOST +ARG DEV_DB_PORT +ARG DEV_DB_NAME # Set working directory (Lambda task root) WORKDIR /var/task # Environment -# ENV DB_HOST=${DEV_DB_HOST} -# ENV DB_PORT=${DEV_DB_PORT} -# ENV DB_NAME=${DEV_DB_NAME} +ENV DB_HOST=${DEV_DB_HOST} +ENV DB_PORT=${DEV_DB_PORT} +ENV DB_NAME=${DEV_DB_NAME} COPY backend/.env.local backend/.env.local diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 11e10997..43b15536 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -4,14 +4,14 @@ import os # from io import BytesIO -# from backend.condition.condition_trigger_request import ConditionTriggerRequest -# from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 -# from backend.condition.processor import process_file -# from utils.logger import setup_logger -# from utils.s3 import read_io_from_s3 +from backend.condition.condition_trigger_request import ConditionTriggerRequest +from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 +from backend.condition.processor import process_file +from utils.logger import setup_logger +from utils.s3 import read_io_from_s3 -# logger = setup_logger() +logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: From 3445060ef2ab832ca7650b4bb87b15e53810c83e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 11:46:58 +0000 Subject: [PATCH 04/50] check db user value --- backend/condition/handler/handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 43b15536..ae6fbf80 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -4,6 +4,7 @@ import os # from io import BytesIO +from backend.app.config import get_settings from backend.condition.condition_trigger_request import ConditionTriggerRequest from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 from backend.condition.processor import process_file @@ -31,6 +32,9 @@ def handler(event: Mapping[str, Any], context: Any) -> None: "hello region", os.getenv("AWS_REGION", "empty region"), ) + settings = get_settings() + print("hello db user", settings.DB_USERNAME) + # uprn_lookup = UprnLookupS3( # bucket="", key="" # ) # TODO: replace with postgres implementation From 04bc8a14075d6da74baf4ada962119915d9ffb25 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 11:51:31 +0000 Subject: [PATCH 05/50] include psycopg2 --- backend/condition/handler/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/condition/handler/requirements.txt b/backend/condition/handler/requirements.txt index 2a54437f..1e259a95 100644 --- a/backend/condition/handler/requirements.txt +++ b/backend/condition/handler/requirements.txt @@ -1,6 +1,7 @@ openpyxl sqlmodel pydantic-settings +psycopg2-binary==2.9.10 # pandas isn't used, but needed for importing from utils.s3 pandas==2.2.2 From f66aa252689f1cecb46a12dd6c39b582ac866808 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 13:19:36 +0000 Subject: [PATCH 06/50] new terraform job to fetch db credentials from secrets manager --- .github/workflows/deploy_terraform.yml | 35 +++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4ac08e41..35002b3a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -66,6 +66,35 @@ jobs: working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan + # ============================================================ + # Fetch DB credentials + # ============================================================ + fetch_db: + needs: determine_stage + runs-on: ubuntu-latest + outputs: + db_username: ${{ steps.get_db.outputs.db_username }} + db_password: ${{ steps.get_db.outputs.db_password }} + + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.DEV_AWS_REGION }} + + - id: get_db + run: | + SECRET=$(aws secretsmanager get-secret-value \ + --secret-id "${{ needs.determine_stage.outputs.stage }}/assessment_model/db_credentials" \ + --query SecretString --output text) + echo "db_username=$(echo $SECRET | jq -r .db_assessment_model_username)" >> $GITHUB_OUTPUT + echo "db_password=$(echo $SECRET | jq -r .db_assessment_model_password)" >> $GITHUB_OUTPUT + + # ============================================================ # 2️⃣ Build Address 2 UPRN image and Push # ============================================================ @@ -157,7 +186,7 @@ jobs: # Deploy Condition ETL Lambda # ============================================================ condition_etl_lambda: - needs: [condition_etl_image, determine_stage] + needs: [condition_etl_image, fetch_db, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: condition-etl @@ -165,6 +194,10 @@ jobs: stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} + environment_vars: | + DB_USERNAME=${{ needs.fetch_db.outputs.db_username }} + DB_PASSWORD=${{ needs.fetch_db.outputs.db_password }} + AWS_DEFAULT_REGION=${{ secrets.DEV_AWS_REGION }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} From 321a4505fbe056c16f8ed0af610f9fd657c8d840 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 13:23:15 +0000 Subject: [PATCH 07/50] no need to get region as env var --- .github/workflows/deploy_terraform.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 35002b3a..df620d04 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -197,7 +197,6 @@ jobs: environment_vars: | DB_USERNAME=${{ needs.fetch_db.outputs.db_username }} DB_PASSWORD=${{ needs.fetch_db.outputs.db_password }} - AWS_DEFAULT_REGION=${{ secrets.DEV_AWS_REGION }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} From 948c2384682440214d5f43a663e98b13a7df4e10 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 13:25:58 +0000 Subject: [PATCH 08/50] include environment_vars on deploy_lambda workflow --- .github/workflows/_deploy_lambda.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index bff106c5..f30a899a 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -23,6 +23,10 @@ on: required: true type: string + environment_vars: + required: false + type: string + secrets: AWS_ACCESS_KEY_ID: required: true From 807d94ca3cb4948c6161efb54b83eb9c22c816ac Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 13:34:08 +0000 Subject: [PATCH 09/50] test again with limited imports --- backend/condition/handler/handler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index ae6fbf80..674d3451 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -5,14 +5,15 @@ import os # from io import BytesIO from backend.app.config import get_settings -from backend.condition.condition_trigger_request import ConditionTriggerRequest -from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 -from backend.condition.processor import process_file -from utils.logger import setup_logger -from utils.s3 import read_io_from_s3 + +# from backend.condition.condition_trigger_request import ConditionTriggerRequest +# from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 +# from backend.condition.processor import process_file +# from utils.logger import setup_logger +# from utils.s3 import read_io_from_s3 -logger = setup_logger() +# logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: From dda05a4f3353161987fe56e6b5b48cf00fe5a149 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 14:34:43 +0000 Subject: [PATCH 10/50] include environment vars in lambda terraform plan --- .github/workflows/_deploy_lambda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index f30a899a..11be11a3 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -88,6 +88,7 @@ jobs: -var="lambda_name=${{ inputs.lambda_name }}" \ -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" \ + -var="environment_vars=${{ inputs.environment_vars }}" \ -out=lambdaplan - name: Terraform Apply From aff3113d525c41b506e93a93179988cdad848e0f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 14:44:50 +0000 Subject: [PATCH 11/50] only load env vars to lambda during deployment if they are provided --- .github/workflows/_deploy_lambda.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 11be11a3..bc83c0af 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -88,7 +88,9 @@ jobs: -var="lambda_name=${{ inputs.lambda_name }}" \ -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" \ - -var="environment_vars=${{ inputs.environment_vars }}" \ + $(if [ -n "${{ inputs.environment_vars }}" ]; then + echo "-var=\"environment_vars=${{ inputs.environment_vars }}\"" + fi) \ -out=lambdaplan - name: Terraform Apply From 78caa1e6ae65858aec93b2421988ee96a19387d7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 14:49:53 +0000 Subject: [PATCH 12/50] format env vars correctly in terraform plan --- .github/workflows/_deploy_lambda.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index bc83c0af..3b25200e 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -83,14 +83,21 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} run: | + ENV_VARS="" + if [ -n "${{ inputs.environment_vars }}" ]; then + # convert multi-line to JSON map: key=value -> "key"="value" + ENV_VARS=$(echo "${{ inputs.environment_vars }}" | \ + awk -F= '{gsub(/"/,"\\\""); printf "\"%s\"=\"%s\",",$1,$2}' | \ + sed 's/,$//') # remove trailing comma + ENV_VARS="-var='environment_vars={${ENV_VARS}}'" + fi + terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" \ - $(if [ -n "${{ inputs.environment_vars }}" ]; then - echo "-var=\"environment_vars=${{ inputs.environment_vars }}\"" - fi) \ + $ENV_VARS \ -out=lambdaplan - name: Terraform Apply From 4c0713b415e88cbcf86b683a37d2625b71976bde Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 15:06:46 +0000 Subject: [PATCH 13/50] include env vars in condition-etl lambda variabls --- .github/workflows/_deploy_lambda.yml | 23 ++++++++----------- .../terraform/lambda/condition-etl/main.tf | 4 ++++ .../lambda/condition-etl/variables.tf | 4 ++++ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 3b25200e..ce4c906a 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -83,22 +83,19 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} run: | - ENV_VARS="" + PLAN_CMD="terraform plan \ + -var=\"stage=${{ inputs.stage }}\" \ + -var=\"lambda_name=${{ inputs.lambda_name }}\" \ + -var=\"ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}\" \ + -var=\"image_digest=${{ inputs.image_digest }}\"" + if [ -n "${{ inputs.environment_vars }}" ]; then - # convert multi-line to JSON map: key=value -> "key"="value" - ENV_VARS=$(echo "${{ inputs.environment_vars }}" | \ - awk -F= '{gsub(/"/,"\\\""); printf "\"%s\"=\"%s\",",$1,$2}' | \ - sed 's/,$//') # remove trailing comma - ENV_VARS="-var='environment_vars={${ENV_VARS}}'" + PLAN_CMD="$PLAN_CMD -var=\"environment_vars=${{ inputs.environment_vars }}\"" fi - terraform plan \ - -var="stage=${{ inputs.stage }}" \ - -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" \ - $ENV_VARS \ - -out=lambdaplan + PLAN_CMD="$PLAN_CMD -out=lambdaplan" + echo "Running: $PLAN_CMD" + eval $PLAN_CMD - name: Terraform Apply working-directory: ${{ inputs.lambda_path }} diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index a421f898..3ab719f7 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -11,4 +11,8 @@ module "lambda" { STAGE = var.stage LOG_LEVEL = "info" } + + # Optional extra environment variables (DB credentials) + environment_vars = var.environment_vars + } diff --git a/infrastructure/terraform/lambda/condition-etl/variables.tf b/infrastructure/terraform/lambda/condition-etl/variables.tf index e4bab243..d4840855 100644 --- a/infrastructure/terraform/lambda/condition-etl/variables.tf +++ b/infrastructure/terraform/lambda/condition-etl/variables.tf @@ -17,6 +17,10 @@ variable "image_digest" { description = "Image digest (sha256:...)" } +variable "environment_vars" { + type = map(string) + default = {} + } locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" From 2fe9e67ea95838ebd796e495b44d6fbc5cdb14ff Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 15:16:15 +0000 Subject: [PATCH 14/50] ensure env vars string is correct format --- .github/workflows/_deploy_lambda.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index ce4c906a..9f2f89a4 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -83,18 +83,24 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} run: | + ENV_VARS="" + if [ -n "${{ inputs.environment_vars }}" ]; then + # Convert multiline "KEY=VALUE" into HCL map format + ENV_VARS="{ $(echo "${{ inputs.environment_vars }}" | awk -F= '{gsub(/"/,"\\\""); printf "%s=\"%s\",",$1,$2}' | sed 's/,$//') }" + fi + PLAN_CMD="terraform plan \ -var=\"stage=${{ inputs.stage }}\" \ -var=\"lambda_name=${{ inputs.lambda_name }}\" \ -var=\"ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}\" \ -var=\"image_digest=${{ inputs.image_digest }}\"" - if [ -n "${{ inputs.environment_vars }}" ]; then - PLAN_CMD="$PLAN_CMD -var=\"environment_vars=${{ inputs.environment_vars }}\"" + if [ -n "$ENV_VARS" ]; then + PLAN_CMD="$PLAN_CMD -var=\"environment_vars=$ENV_VARS\"" fi PLAN_CMD="$PLAN_CMD -out=lambdaplan" - echo "Running: $PLAN_CMD" + # echo "Running: $PLAN_CMD" eval $PLAN_CMD - name: Terraform Apply From d44fdcd01c5f437abdaa9bf88be8564e544380ea Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 15:22:36 +0000 Subject: [PATCH 15/50] try again --- .github/workflows/_deploy_lambda.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 9f2f89a4..35275341 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -85,8 +85,11 @@ jobs: run: | ENV_VARS="" if [ -n "${{ inputs.environment_vars }}" ]; then - # Convert multiline "KEY=VALUE" into HCL map format - ENV_VARS="{ $(echo "${{ inputs.environment_vars }}" | awk -F= '{gsub(/"/,"\\\""); printf "%s=\"%s\",",$1,$2}' | sed 's/,$//') }" + # Convert multiline KEY=VALUE into JSON + ENV_VARS=$(echo "${{ inputs.environment_vars }}" | \ + jq -Rn ' + [inputs | split("=")] | + { (.[0]): .[1] }' | jq -s add | jq -c .) fi PLAN_CMD="terraform plan \ From f033b144c23ba516d6ccf3ebdcffd9d85f7933d4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 16:04:57 +0000 Subject: [PATCH 16/50] including environment in vars not environment_vars --- backend/condition/handler/handler.py | 4 ++++ infrastructure/terraform/lambda/condition-etl/main.tf | 5 ++--- infrastructure/terraform/lambda/condition-etl/variables.tf | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 674d3451..33d6559c 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -33,6 +33,10 @@ def handler(event: Mapping[str, Any], context: Any) -> None: "hello region", os.getenv("AWS_REGION", "empty region"), ) + print( + "hello Dan:", + os.getenv("DAN", "empty dan"), + ) settings = get_settings() print("hello db user", settings.DB_USERNAME) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 3ab719f7..2e2ee69b 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -10,9 +10,8 @@ module "lambda" { environment = { STAGE = var.stage LOG_LEVEL = "info" + DB_USERNAME = var.environment_vars.DB_USERNAME + DAN = "hello" } - # Optional extra environment variables (DB credentials) - environment_vars = var.environment_vars - } diff --git a/infrastructure/terraform/lambda/condition-etl/variables.tf b/infrastructure/terraform/lambda/condition-etl/variables.tf index d4840855..3f5004a9 100644 --- a/infrastructure/terraform/lambda/condition-etl/variables.tf +++ b/infrastructure/terraform/lambda/condition-etl/variables.tf @@ -18,9 +18,9 @@ variable "image_digest" { } variable "environment_vars" { - type = map(string) - default = {} - } + type = map(string) + default = {} +} locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" From bae1735e23c00e653803f9438d60d65c30af7be5 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 6 Feb 2026 16:22:53 +0000 Subject: [PATCH 17/50] try using shared resource to fetch secrets --- .github/workflows/_deploy_lambda.yml | 34 ++++-------- .github/workflows/deploy_terraform.yml | 52 ++++++++++--------- .../terraform/lambda/condition-etl/main.tf | 23 +++++--- .../lambda/condition-etl/variables.tf | 4 -- 4 files changed, 53 insertions(+), 60 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 35275341..f883636e 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -23,9 +23,9 @@ on: required: true type: string - environment_vars: - required: false - type: string + # environment_vars: + # required: false + # type: string secrets: AWS_ACCESS_KEY_ID: @@ -83,28 +83,12 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} run: | - ENV_VARS="" - if [ -n "${{ inputs.environment_vars }}" ]; then - # Convert multiline KEY=VALUE into JSON - ENV_VARS=$(echo "${{ inputs.environment_vars }}" | \ - jq -Rn ' - [inputs | split("=")] | - { (.[0]): .[1] }' | jq -s add | jq -c .) - fi - - PLAN_CMD="terraform plan \ - -var=\"stage=${{ inputs.stage }}\" \ - -var=\"lambda_name=${{ inputs.lambda_name }}\" \ - -var=\"ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}\" \ - -var=\"image_digest=${{ inputs.image_digest }}\"" - - if [ -n "$ENV_VARS" ]; then - PLAN_CMD="$PLAN_CMD -var=\"environment_vars=$ENV_VARS\"" - fi - - PLAN_CMD="$PLAN_CMD -out=lambdaplan" - # echo "Running: $PLAN_CMD" - eval $PLAN_CMD + terraform plan \ + -var="stage=${{ inputs.stage }}" \ + -var="lambda_name=${{ inputs.lambda_name }}" \ + -var="ecr_repo_url=${{ inputs.ecr_repo }}" \ + -var="image_digest=${{ inputs.image_digest }}" \ + -out=lambdaplan - name: Terraform Apply working-directory: ${{ inputs.lambda_path }} diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index df620d04..52f64f68 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -69,30 +69,30 @@ jobs: # ============================================================ # Fetch DB credentials # ============================================================ - fetch_db: - needs: determine_stage - runs-on: ubuntu-latest - outputs: - db_username: ${{ steps.get_db.outputs.db_username }} - db_password: ${{ steps.get_db.outputs.db_password }} + # fetch_db: + # needs: determine_stage + # runs-on: ubuntu-latest + # outputs: + # db_username: ${{ steps.get_db.outputs.db_username }} + # db_password: ${{ steps.get_db.outputs.db_password }} - steps: - - uses: actions/checkout@v4 + # steps: + # - uses: actions/checkout@v4 - - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.DEV_AWS_REGION }} + # - name: Configure AWS + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.DEV_AWS_REGION }} - - id: get_db - run: | - SECRET=$(aws secretsmanager get-secret-value \ - --secret-id "${{ needs.determine_stage.outputs.stage }}/assessment_model/db_credentials" \ - --query SecretString --output text) - echo "db_username=$(echo $SECRET | jq -r .db_assessment_model_username)" >> $GITHUB_OUTPUT - echo "db_password=$(echo $SECRET | jq -r .db_assessment_model_password)" >> $GITHUB_OUTPUT + # - id: get_db + # run: | + # SECRET=$(aws secretsmanager get-secret-value \ + # --secret-id "${{ needs.determine_stage.outputs.stage }}/assessment_model/db_credentials" \ + # --query SecretString --output text) + # echo "db_username=$(echo $SECRET | jq -r .db_assessment_model_username)" >> $GITHUB_OUTPUT + # echo "db_password=$(echo $SECRET | jq -r .db_assessment_model_password)" >> $GITHUB_OUTPUT # ============================================================ @@ -186,7 +186,8 @@ jobs: # Deploy Condition ETL Lambda # ============================================================ condition_etl_lambda: - needs: [condition_etl_image, fetch_db, determine_stage] + # needs: [condition_etl_image, fetch_db, determine_stage] + needs: [condition_etl_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: condition-etl @@ -194,9 +195,10 @@ jobs: stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} - environment_vars: | - DB_USERNAME=${{ needs.fetch_db.outputs.db_username }} - DB_PASSWORD=${{ needs.fetch_db.outputs.db_password }} + # environment_vars: ${{ toJSON({ + # DB_USERNAME: needs.fetch_db.outputs.db_username, + # DB_PASSWORD: needs.fetch_db.outputs.db_password + # }) }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 2e2ee69b..dda57385 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -1,3 +1,12 @@ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + + module "lambda" { source = "../modules/lambda_with_sqs" @@ -7,11 +16,13 @@ module "lambda" { image_uri = local.image_uri - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - DB_USERNAME = var.environment_vars.DB_USERNAME - DAN = "hello" - } + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + }, + ) } diff --git a/infrastructure/terraform/lambda/condition-etl/variables.tf b/infrastructure/terraform/lambda/condition-etl/variables.tf index 3f5004a9..e4bab243 100644 --- a/infrastructure/terraform/lambda/condition-etl/variables.tf +++ b/infrastructure/terraform/lambda/condition-etl/variables.tf @@ -17,10 +17,6 @@ variable "image_digest" { description = "Image digest (sha256:...)" } -variable "environment_vars" { - type = map(string) - default = {} -} locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" From daaa3c560ff010313253b7d6cd212bb2843cfee9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 09:20:50 +0000 Subject: [PATCH 18/50] revert changes to _deploy_lambda --- .github/workflows/_deploy_lambda.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index f883636e..bff106c5 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -23,10 +23,6 @@ on: required: true type: string - # environment_vars: - # required: false - # type: string - secrets: AWS_ACCESS_KEY_ID: required: true @@ -86,7 +82,7 @@ jobs: terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ inputs.ecr_repo }}" \ + -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" \ -out=lambdaplan From f41da9e93fdc2e5fc80a45901d3c59d300f2ec17 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 09:28:17 +0000 Subject: [PATCH 19/50] remove commented out deployment jobs --- .github/workflows/deploy_terraform.yml | 34 -------------------------- 1 file changed, 34 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 52f64f68..4ac08e41 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -66,35 +66,6 @@ jobs: working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan - # ============================================================ - # Fetch DB credentials - # ============================================================ - # fetch_db: - # needs: determine_stage - # runs-on: ubuntu-latest - # outputs: - # db_username: ${{ steps.get_db.outputs.db_username }} - # db_password: ${{ steps.get_db.outputs.db_password }} - - # steps: - # - uses: actions/checkout@v4 - - # - name: Configure AWS - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.DEV_AWS_REGION }} - - # - id: get_db - # run: | - # SECRET=$(aws secretsmanager get-secret-value \ - # --secret-id "${{ needs.determine_stage.outputs.stage }}/assessment_model/db_credentials" \ - # --query SecretString --output text) - # echo "db_username=$(echo $SECRET | jq -r .db_assessment_model_username)" >> $GITHUB_OUTPUT - # echo "db_password=$(echo $SECRET | jq -r .db_assessment_model_password)" >> $GITHUB_OUTPUT - - # ============================================================ # 2️⃣ Build Address 2 UPRN image and Push # ============================================================ @@ -186,7 +157,6 @@ jobs: # Deploy Condition ETL Lambda # ============================================================ condition_etl_lambda: - # needs: [condition_etl_image, fetch_db, determine_stage] needs: [condition_etl_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: @@ -195,10 +165,6 @@ jobs: stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} - # environment_vars: ${{ toJSON({ - # DB_USERNAME: needs.fetch_db.outputs.db_username, - # DB_PASSWORD: needs.fetch_db.outputs.db_password - # }) }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} From 9a9ea9fbaaf7f769b8243f0eebbd67f720574b26 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 09:36:41 +0000 Subject: [PATCH 20/50] remove unused bits of config --- .github/workflows/_build_image.yml | 12 +---- backend/condition/handler/handler.py | 74 ++++++++++------------------ 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 41677b1f..d5e028b2 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -32,13 +32,7 @@ on: required: true AWS_REGION: required: true - DEV_DB_HOST: - required: false - DEV_DB_PORT: - required: false - DEV_DB_NAME: - required: false - + jobs: build: runs-on: ubuntu-latest @@ -87,10 +81,6 @@ jobs: temp=$(eval echo "$line") BUILD_ARGS="$BUILD_ARGS --build-arg $temp" done <<< "${{ inputs.build_args }}" - - echo "dev db host: $DEV_DB_HOST" - echo "dev db port: $DEV_DB_PORT" - echo "dev db name: $DEV_DB_NAME" docker build \ -f ${{ inputs.dockerfile_path }} \ diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 33d6559c..fabfc84c 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -1,64 +1,42 @@ -# import json +import json from typing import Mapping, Any import os -# from io import BytesIO +from io import BytesIO from backend.app.config import get_settings -# from backend.condition.condition_trigger_request import ConditionTriggerRequest -# from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 -# from backend.condition.processor import process_file -# from utils.logger import setup_logger -# from utils.s3 import read_io_from_s3 +from backend.condition.condition_trigger_request import ConditionTriggerRequest +from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 +from backend.condition.processor import process_file +from utils.logger import setup_logger +from utils.s3 import read_io_from_s3 -# logger = setup_logger() +logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: - print( - "hello DB HOST:", - os.getenv("DB_HOST", "empty db host"), - ) - print( - "hello DB PORT:", - os.getenv("DB_PORT", "empty db port"), - ) - print( - "hello DB NAME:", - os.getenv("DB_NAME", "empty name"), - ) - print( - "hello region", - os.getenv("AWS_REGION", "empty region"), - ) - print( - "hello Dan:", - os.getenv("DAN", "empty dan"), - ) - settings = get_settings() - print("hello db user", settings.DB_USERNAME) - # uprn_lookup = UprnLookupS3( - # bucket="", key="" - # ) # TODO: replace with postgres implementation + uprn_lookup = UprnLookupS3( + bucket="", key="" + ) # TODO: replace with postgres implementation - # for record in event.get("Records", []): - # try: - # body_dict = json.loads(record["body"]) - # payload = ConditionTriggerRequest.model_validate(body_dict) + for record in event.get("Records", []): + try: + body_dict = json.loads(record["body"]) + payload = ConditionTriggerRequest.model_validate(body_dict) - # file_bytes: BytesIO = read_io_from_s3( - # bucket_name=payload.trigger_file_bucket, - # file_key=payload.trigger_file_key, - # ) + file_bytes: BytesIO = read_io_from_s3( + bucket_name=payload.trigger_file_bucket, + file_key=payload.trigger_file_key, + ) - # process_file( - # file_stream=file_bytes, - # file_type=payload.file_type, - # uprn_lookup=uprn_lookup, - # ) + process_file( + file_stream=file_bytes, + file_type=payload.file_type, + uprn_lookup=uprn_lookup, + ) - # except Exception as e: - # logger.error(f"Failed to process record: {e}") + except Exception as e: + logger.error(f"Failed to process record: {e}") From 6dbb8149fd1998cb7ac058ab0be455f4e475fd77 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 09:37:43 +0000 Subject: [PATCH 21/50] reinstate used github secrets --- .github/workflows/_build_image.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index d5e028b2..408c0319 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -32,7 +32,13 @@ on: required: true AWS_REGION: required: true - + DEV_DB_HOST: + required: false + DEV_DB_PORT: + required: false + DEV_DB_NAME: + required: false + jobs: build: runs-on: ubuntu-latest @@ -81,7 +87,7 @@ jobs: temp=$(eval echo "$line") BUILD_ARGS="$BUILD_ARGS --build-arg $temp" done <<< "${{ inputs.build_args }}" - + docker build \ -f ${{ inputs.dockerfile_path }} \ $BUILD_ARGS \ From 0e4da14673086f8f4087cf4b5723ece981dd2903 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 09:56:41 +0000 Subject: [PATCH 22/50] get lookup file bucket and key from payload per message --- backend/condition/condition_trigger_request.py | 6 ++++-- backend/condition/handler/handler.py | 16 ++++++++-------- backend/condition/processor.py | 6 ++++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/backend/condition/condition_trigger_request.py b/backend/condition/condition_trigger_request.py index 1bea6a0d..130d9896 100644 --- a/backend/condition/condition_trigger_request.py +++ b/backend/condition/condition_trigger_request.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import Optional from pydantic import BaseModel @@ -12,5 +13,6 @@ class ConditionTriggerRequest(BaseModel): file_type: ConditionFileType trigger_file_bucket: str # TODO: get this from settings trigger_file_key: str - uprn_lookup_file_bucket: str # TODO: get this from settings - uprn_lookup_file_key: str + + uprn_lookup_file_bucket: Optional[str] = None # TODO: get this from settings + uprn_lookup_file_key: Optional[str] = None diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index fabfc84c..bb3b68ed 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -1,11 +1,7 @@ import json from typing import Mapping, Any -import os - from io import BytesIO -from backend.app.config import get_settings - from backend.condition.condition_trigger_request import ConditionTriggerRequest from backend.condition.lookups.uprn_lookup_s3 import UprnLookupS3 from backend.condition.processor import process_file @@ -18,15 +14,19 @@ logger = setup_logger() def handler(event: Mapping[str, Any], context: Any) -> None: - uprn_lookup = UprnLookupS3( - bucket="", key="" - ) # TODO: replace with postgres implementation - for record in event.get("Records", []): try: body_dict = json.loads(record["body"]) payload = ConditionTriggerRequest.model_validate(body_dict) + if payload.uprn_lookup_file_bucket and payload.uprn_lookup_file_key: + uprn_lookup = UprnLookupS3( + bucket=payload.uprn_lookup_file_bucket, + key=payload.uprn_lookup_file_key, + ) # TODO: replace with postgres implementation + else: + uprn_lookup = None + file_bytes: BytesIO = read_io_from_s3( bucket_name=payload.trigger_file_bucket, file_key=payload.trigger_file_key, diff --git a/backend/condition/processor.py b/backend/condition/processor.py index 70ce2df9..8d281561 100644 --- a/backend/condition/processor.py +++ b/backend/condition/processor.py @@ -1,4 +1,4 @@ -from typing import Any, BinaryIO, List +from typing import Any, BinaryIO, List, Optional from datetime import datetime from backend.condition.condition_trigger_request import ConditionFileType @@ -14,7 +14,9 @@ logger = setup_logger() def process_file( - file_stream: BinaryIO, file_type: ConditionFileType, uprn_lookup: UprnLookup + file_stream: BinaryIO, + file_type: ConditionFileType, + uprn_lookup: Optional[UprnLookup], ) -> None: # Instantiation parser: Parser = select_parser(file_type, uprn_lookup) From e1835b2cfacf5ed53f395893e2aa95c24c0ccd51 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 10:16:18 +0000 Subject: [PATCH 23/50] remove duplicates from .env.local --- backend/.env.local | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/backend/.env.local b/backend/.env.local index 4e401ed9..5b77f243 100644 --- a/backend/.env.local +++ b/backend/.env.local @@ -19,22 +19,4 @@ PLAN_TRIGGER_BUCKET=test DATA_BUCKET=test EPC_AUTH_TOKEN=test ENGINE_SQS_URL=test -ENERGY_ASSESSMENTS_BUCKET=test -API_KEY=test -SECRET_KEY=test -ENVIRONMENT=test -DATA_BUCKET=test -PLAN_TRIGGER_BUCKET=test -ENGINE_SQS_URL=test -GOOGLE_SOLAR_API_KEY=test -DB_HOST=test -DB_PASSWORD=test -DB_USERNAME=test -DB_PORT=test -DB_NAME=test -SAP_PREDICTIONS_BUCKET=test -CARBON_PREDICTIONS_BUCKET=test -HEAT_PREDICTIONS_BUCKET=test -HEATING_KWH_PREDICTIONS_BUCKET=test -HOTWATER_KWH_PREDICTIONS_BUCKET=test ENERGY_ASSESSMENTS_BUCKET=test \ No newline at end of file From 092d95fa9d211e3f5b214415ca19b783933f28fd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 11:45:59 +0000 Subject: [PATCH 24/50] create new s3 bucket for condition data --- infrastructure/terraform/shared/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 4b22b302..9b2ded3c 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -335,4 +335,12 @@ module "postcode_splitter_registry" { name = "postcode_splitter" stage = var.stage +} + +################################################ +# Conidition data – S3 bucket +################################################ +module "condition_data_bucket" { + source = "../modules/s3" + bucket_name = "condition-data" } \ No newline at end of file From a2cdf0eb57d562a1a6b37cf4e20322e84db3e59e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 12:02:13 +0000 Subject: [PATCH 25/50] fix bucketname typo --- infrastructure/terraform/shared/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 9b2ded3c..cd015952 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -342,5 +342,5 @@ module "postcode_splitter_registry" { ################################################ module "condition_data_bucket" { source = "../modules/s3" - bucket_name = "condition-data" + bucketname = "condition-data-${var.stage}" } \ No newline at end of file From 56e78350fa9cb29c618e4ebcc1373a56a27b8a4a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 12:04:24 +0000 Subject: [PATCH 26/50] configure allowed_origins --- infrastructure/terraform/shared/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index cd015952..d1d48aec 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -343,4 +343,5 @@ module "postcode_splitter_registry" { module "condition_data_bucket" { source = "../modules/s3" bucketname = "condition-data-${var.stage}" + allowed_origins = var.allowed_origins } \ No newline at end of file From 2e424e227eef5e5e1a91c111b1ce10074e0b6df1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 12:19:17 +0000 Subject: [PATCH 27/50] apply shared stack to generate new s3 bucket --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4ac08e41..0e541c9f 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -62,7 +62,7 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply - if: env.STAGE == 'prod' + # if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan From db0431f413fe5734331464c65f125b7ed0c94e8e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 12:20:44 +0000 Subject: [PATCH 28/50] recomment env.stage == 'prod' --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 0e541c9f..4ac08e41 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -62,7 +62,7 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply - # if: env.STAGE == 'prod' + if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan From 68b12c73445b228e2c3c852a459cdf26b564c138 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 12:49:32 +0000 Subject: [PATCH 29/50] trigger rebuild and deploy --- backend/condition/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/condition/processor.py b/backend/condition/processor.py index 8d281561..650442aa 100644 --- a/backend/condition/processor.py +++ b/backend/condition/processor.py @@ -45,4 +45,4 @@ def process_file( # persistence.bulk_insert_surveys(property_condition_surveys) - logger.info(f"[processor] Finished loading surveys to database") + # logger.info(f"[processor] Finished loading surveys to database") From f3e77beefdacb9a2a4ef9d26035d9c38458901bd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 14:13:06 +0000 Subject: [PATCH 30/50] generate new role for reading from s3 bucket and attach it to the lambda --- .../terraform/lambda/condition-etl/main.tf | 5 +++++ infrastructure/terraform/shared/main.tf | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index dda57385..ecd52644 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -26,3 +26,8 @@ module "lambda" { ) } + +resource "aws_iam_role_policy_attachment" "attach_condition_etl_s3_read" { + role = module.lambda.role.role_name + policy_arn = module.shared.condition_etl_s3_read_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index d1d48aec..3b1404ed 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -344,4 +344,23 @@ module "condition_data_bucket" { source = "../modules/s3" bucketname = "condition-data-${var.stage}" allowed_origins = var.allowed_origins +} + +resource "aws_iam_policy" "condition_etl_s3_read" { + name = "ConditionETLReadS3" + description = "Allow Lambda to read objects from condition-data-${var.stage}" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "arn:aws:s3:::condition-data-${var.stage}/*" + } + ] + }) +} + +output "condition_etl_s3_read_arn" { + value = aws_iam_policy.condition_etl_s3_read.arn } \ No newline at end of file From dfff8adc3d9f34b4a6980395a8736be0eeff040a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 14:20:22 +0000 Subject: [PATCH 31/50] Apply: generate new role for reading from s3 bucket and attach it to the lambda --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4ac08e41..0e541c9f 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -62,7 +62,7 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply - if: env.STAGE == 'prod' + # if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan From 1e1db67f3c071a93a5f067d270561b5af71b7b2a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 14:34:29 +0000 Subject: [PATCH 32/50] import shared module into condition-etl tf --- infrastructure/terraform/lambda/condition-etl/main.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index ecd52644..69536881 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -6,6 +6,11 @@ locals { db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) } +module "shared" { + source = "../../shared" + stage = var.stage + allowed_origins = var.allowed_origins +} module "lambda" { source = "../modules/lambda_with_sqs" From 658761933b4759e2a3434256da39fb07ac52dbe3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 14:56:46 +0000 Subject: [PATCH 33/50] read policy ARN from shared stack state --- .../terraform/lambda/condition-etl/main.tf | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 69536881..fa00185f 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -6,10 +6,13 @@ locals { db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) } -module "shared" { - source = "../../shared" - stage = var.stage - allowed_origins = var.allowed_origins +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "condition-etl-terraform-state" + key = "shared/terraform.tfstate" + region = "eu-west-2" + } } module "lambda" { @@ -34,5 +37,5 @@ module "lambda" { resource "aws_iam_role_policy_attachment" "attach_condition_etl_s3_read" { role = module.lambda.role.role_name - policy_arn = module.shared.condition_etl_s3_read_arn + policy_arn = data.terraform_remote_state.shared.outputs.condition_etl_s3_read_arn } \ No newline at end of file From 7630067d8d52e7ed881fa5053401943ebdbc25b0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 15:05:01 +0000 Subject: [PATCH 34/50] expose role name from lambda_with_sqs --- .../terraform/lambda/modules/lambda_with_sqs/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf index 3816c206..065fb790 100644 --- a/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf +++ b/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf @@ -6,6 +6,10 @@ module "role" { name = "${var.name}-lambda-${var.stage}" } +output "role_name" { + value = module.role.role_name +} + ############################################ # SQS queue + DLQ ############################################ From de6bb935fe1043fe4c8641a71c31aa6c15e6f283 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 15:10:29 +0000 Subject: [PATCH 35/50] correct role import path in condition etl tf --- infrastructure/terraform/lambda/condition-etl/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index fa00185f..8d9f2171 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -36,6 +36,6 @@ module "lambda" { } resource "aws_iam_role_policy_attachment" "attach_condition_etl_s3_read" { - role = module.lambda.role.role_name + role = module.lambda.role_name policy_arn = data.terraform_remote_state.shared.outputs.condition_etl_s3_read_arn } \ No newline at end of file From a1bb768dfe6f7ce4b512a3935fd44b3324e2a150 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 15:20:06 +0000 Subject: [PATCH 36/50] correct path to tfstate file --- infrastructure/terraform/lambda/condition-etl/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 8d9f2171..a81444f9 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -10,7 +10,8 @@ data "terraform_remote_state" "shared" { backend = "s3" config = { bucket = "condition-etl-terraform-state" - key = "shared/terraform.tfstate" + key = "env:/dev/terraform.tfstate" # TODO: dont hardcode this + # key = "shared/terraform.tfstate" region = "eu-west-2" } } From 888d46f2756c172d18ac60edc0a7e0db7bb15f66 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 15:52:15 +0000 Subject: [PATCH 37/50] correct tfstate bucket and path --- infrastructure/terraform/lambda/condition-etl/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index a81444f9..98e97a08 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -9,9 +9,8 @@ locals { data "terraform_remote_state" "shared" { backend = "s3" config = { - bucket = "condition-etl-terraform-state" + bucket = "assessment-model-terraform-state" key = "env:/dev/terraform.tfstate" # TODO: dont hardcode this - # key = "shared/terraform.tfstate" region = "eu-west-2" } } From 1f1e077b4974e76acbe43bf9fd865c55f4616afd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 16:05:42 +0000 Subject: [PATCH 38/50] use var.stage when defining terraform state s3 key --- infrastructure/terraform/lambda/condition-etl/main.tf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 98e97a08..0128f975 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -2,19 +2,20 @@ data "aws_secretsmanager_secret_version" "db_credentials" { secret_id = "${var.stage}/assessment_model/db_credentials" } -locals { - db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) -} - data "terraform_remote_state" "shared" { backend = "s3" config = { bucket = "assessment-model-terraform-state" - key = "env:/dev/terraform.tfstate" # TODO: dont hardcode this + key = "env:/${var.stage}/terraform.tfstate" # TODO: dont hardcode this region = "eu-west-2" } } +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + + module "lambda" { source = "../modules/lambda_with_sqs" From d1e9e9f7ab3d279ac4c328228d4f59e02b31b2af Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 16:10:00 +0000 Subject: [PATCH 39/50] uncomment if: env.STAGE == 'prod' --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 0e541c9f..4ac08e41 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -62,7 +62,7 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply - # if: env.STAGE == 'prod' + if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan From 3fa51e3c3abaf11b0bda44cc7b8c152ec6f45f19 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 16:32:25 +0000 Subject: [PATCH 40/50] extend lambda timeout to 3 minutes --- infrastructure/terraform/lambda/condition-etl/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 0128f975..4219f209 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -23,6 +23,7 @@ module "lambda" { stage = var.stage image_uri = local.image_uri + timeout = 180 environment = merge( From 9b6f99929ea32939959218bc174e58349a0fd75e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 17:00:59 +0000 Subject: [PATCH 41/50] add logging --- backend/condition/handler/handler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index bb3b68ed..08d67aa8 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -17,20 +17,29 @@ def handler(event: Mapping[str, Any], context: Any) -> None: for record in event.get("Records", []): try: body_dict = json.loads(record["body"]) + logger.info("Validating request body") payload = ConditionTriggerRequest.model_validate(body_dict) + logger.info("Successfully validated request body") + if payload.uprn_lookup_file_bucket and payload.uprn_lookup_file_key: + logger.info("Getting UPRN lookup file from s3") uprn_lookup = UprnLookupS3( bucket=payload.uprn_lookup_file_bucket, key=payload.uprn_lookup_file_key, ) # TODO: replace with postgres implementation + logger.info("Successfully got UPRN lookup file from s3") else: uprn_lookup = None + logger.info("Getting conditions data from s3") file_bytes: BytesIO = read_io_from_s3( bucket_name=payload.trigger_file_bucket, file_key=payload.trigger_file_key, ) + logger.info( + "Successfully got conditions data from s3. Moving on to process file..." + ) process_file( file_stream=file_bytes, From cd0ae37d4070c11c36320f9a3fe77c7c7c1d23fa Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 9 Feb 2026 17:18:45 +0000 Subject: [PATCH 42/50] more logging --- backend/condition/parsing/peabody_parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/condition/parsing/peabody_parser.py b/backend/condition/parsing/peabody_parser.py index 126bcfea..acd30084 100644 --- a/backend/condition/parsing/peabody_parser.py +++ b/backend/condition/parsing/peabody_parser.py @@ -24,10 +24,13 @@ class PeabodyParser(Parser): file_stream: BinaryIO, ) -> Any: wb: Workbook = load_workbook(file_stream) + logger.info("[PeabodyParser] Parsing assets") assets = PeabodyParser._parse_assets(wb) + logger.info("[PeabodyParser] Successfully parsed assets") + logger.info("[PeabodyParser] Parsing UPRN lookup") location_ref_to_uprn_map = self.uprn_lookup.get_property_ref_to_uprn_lookup() - + logger.info("[PeabodyParser] Successfully parsed UPRN lookup") return PeabodyParser._group_assets_into_properties( assets=assets, location_ref_to_uprn_map=location_ref_to_uprn_map, From b026a7ae9e7d8f97037e9c2a2a9653e212f8ff12 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:02:44 +0000 Subject: [PATCH 43/50] log db session but dont write to db --- .../persistence/condition_postgres.py | 19 ++++++++++--------- backend/condition/processor.py | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/condition/persistence/condition_postgres.py b/backend/condition/persistence/condition_postgres.py index 9d7895f0..9b9ce2e0 100644 --- a/backend/condition/persistence/condition_postgres.py +++ b/backend/condition/persistence/condition_postgres.py @@ -31,17 +31,18 @@ class ConditionPostgres: ) with db_session() as session: - for start in range(0, total, batch_size): - end = min(start + batch_size, total) - batch = survey_models[start:end] + logger.info("Successfully made connection to database:", session) + # for start in range(0, total, batch_size): + # end = min(start + batch_size, total) + # batch = survey_models[start:end] - t0: float = time.perf_counter() - ConditionPostgres._insert_surveys_batch(batch, session) - elapsed: float = time.perf_counter() - t0 + # t0: float = time.perf_counter() + # ConditionPostgres._insert_surveys_batch(batch, session) + # elapsed: float = time.perf_counter() - t0 - logger.info( - f"Inserted batch {start} - {end} ({len(batch)} surveys) in {elapsed} seconds", - ) + # logger.info( + # f"Inserted batch {start} - {end} ({len(batch)} surveys) in {elapsed} seconds", + # ) @staticmethod def map_survey_to_model( diff --git a/backend/condition/processor.py b/backend/condition/processor.py index 650442aa..b7c7d2b6 100644 --- a/backend/condition/processor.py +++ b/backend/condition/processor.py @@ -43,6 +43,6 @@ def process_file( f"[processor] Finished mapping {len(property_condition_surveys)} properties. Writing to database..." ) - # persistence.bulk_insert_surveys(property_condition_surveys) + persistence.bulk_insert_surveys(property_condition_surveys) - # logger.info(f"[processor] Finished loading surveys to database") + logger.info(f"[processor] Finished loading surveys to database") From c0534d9546460856f4e4ff3c35941a899f6a6e8c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:16:52 +0000 Subject: [PATCH 44/50] more logs --- backend/condition/condition_trigger_request.py | 9 +++++++++ backend/condition/parsing/peabody_parser.py | 8 +++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/condition/condition_trigger_request.py b/backend/condition/condition_trigger_request.py index 130d9896..0477d044 100644 --- a/backend/condition/condition_trigger_request.py +++ b/backend/condition/condition_trigger_request.py @@ -16,3 +16,12 @@ class ConditionTriggerRequest(BaseModel): uprn_lookup_file_bucket: Optional[str] = None # TODO: get this from settings uprn_lookup_file_key: Optional[str] = None + + +# { +# "file_type": "Peabody", +# "trigger_file_bucket": "condition-data-dev", +# "trigger_file_key": "input/peabody/2026_01_06 - Peabody - Stock Condition Data - Survey Records - D Lower.xlsx", +# "uprn_lookup_file_bucket": "condition-data-dev", +# "uprn_lookup_file_key": "input/peabody/uprn-lookup/PeabodyPropertymatched_Dec25_propref_UPRN.csv" +# } diff --git a/backend/condition/parsing/peabody_parser.py b/backend/condition/parsing/peabody_parser.py index acd30084..9a38c790 100644 --- a/backend/condition/parsing/peabody_parser.py +++ b/backend/condition/parsing/peabody_parser.py @@ -23,12 +23,14 @@ class PeabodyParser(Parser): self, file_stream: BinaryIO, ) -> Any: + logger.info("[PeabodyParser] Loading workbook...") wb: Workbook = load_workbook(file_stream) - logger.info("[PeabodyParser] Parsing assets") + logger.info("[PeabodyParser] Successfully loaded workbook. Parsing assets...") assets = PeabodyParser._parse_assets(wb) - logger.info("[PeabodyParser] Successfully parsed assets") + logger.info( + "[PeabodyParser] Successfully parsed assets. Parsing UPRN lookup..." + ) - logger.info("[PeabodyParser] Parsing UPRN lookup") location_ref_to_uprn_map = self.uprn_lookup.get_property_ref_to_uprn_lookup() logger.info("[PeabodyParser] Successfully parsed UPRN lookup") return PeabodyParser._group_assets_into_properties( From 06539a787dda6f440b25e1a647e56ef0d72d11e0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:30:39 +0000 Subject: [PATCH 45/50] use streaming mode in openpyxl for peabody parser --- backend/condition/parsing/peabody_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/condition/parsing/peabody_parser.py b/backend/condition/parsing/peabody_parser.py index 9a38c790..921f24d4 100644 --- a/backend/condition/parsing/peabody_parser.py +++ b/backend/condition/parsing/peabody_parser.py @@ -23,8 +23,9 @@ class PeabodyParser(Parser): self, file_stream: BinaryIO, ) -> Any: + file_stream.seek(0) logger.info("[PeabodyParser] Loading workbook...") - wb: Workbook = load_workbook(file_stream) + wb: Workbook = load_workbook(file_stream, read_only=True, data_only=True) logger.info("[PeabodyParser] Successfully loaded workbook. Parsing assets...") assets = PeabodyParser._parse_assets(wb) logger.info( From d0847681c182d9c1568e7ef966c62d0ed0c751b0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:42:29 +0000 Subject: [PATCH 46/50] tidy up logs --- backend/condition/handler/handler.py | 12 ++++++------ backend/condition/parsing/peabody_parser.py | 14 ++++++++------ backend/condition/processor.py | 3 +++ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/backend/condition/handler/handler.py b/backend/condition/handler/handler.py index 08d67aa8..2f3616a4 100644 --- a/backend/condition/handler/handler.py +++ b/backend/condition/handler/handler.py @@ -17,27 +17,27 @@ def handler(event: Mapping[str, Any], context: Any) -> None: for record in event.get("Records", []): try: body_dict = json.loads(record["body"]) - logger.info("Validating request body") + logger.debug("Validating request body") payload = ConditionTriggerRequest.model_validate(body_dict) - logger.info("Successfully validated request body") + logger.debug("Successfully validated request body") if payload.uprn_lookup_file_bucket and payload.uprn_lookup_file_key: - logger.info("Getting UPRN lookup file from s3") + logger.debug("Getting UPRN lookup file from s3") uprn_lookup = UprnLookupS3( bucket=payload.uprn_lookup_file_bucket, key=payload.uprn_lookup_file_key, ) # TODO: replace with postgres implementation - logger.info("Successfully got UPRN lookup file from s3") + logger.debug("Successfully got UPRN lookup file from s3") else: uprn_lookup = None - logger.info("Getting conditions data from s3") + logger.debug("Getting conditions data from s3") file_bytes: BytesIO = read_io_from_s3( bucket_name=payload.trigger_file_bucket, file_key=payload.trigger_file_key, ) - logger.info( + logger.debug( "Successfully got conditions data from s3. Moving on to process file..." ) diff --git a/backend/condition/parsing/peabody_parser.py b/backend/condition/parsing/peabody_parser.py index 921f24d4..4620ba82 100644 --- a/backend/condition/parsing/peabody_parser.py +++ b/backend/condition/parsing/peabody_parser.py @@ -24,16 +24,16 @@ class PeabodyParser(Parser): file_stream: BinaryIO, ) -> Any: file_stream.seek(0) - logger.info("[PeabodyParser] Loading workbook...") + logger.debug("[PeabodyParser] Loading workbook...") wb: Workbook = load_workbook(file_stream, read_only=True, data_only=True) - logger.info("[PeabodyParser] Successfully loaded workbook. Parsing assets...") + logger.debug("[PeabodyParser] Successfully loaded workbook. Parsing assets...") assets = PeabodyParser._parse_assets(wb) - logger.info( + logger.debug( "[PeabodyParser] Successfully parsed assets. Parsing UPRN lookup..." ) location_ref_to_uprn_map = self.uprn_lookup.get_property_ref_to_uprn_lookup() - logger.info("[PeabodyParser] Successfully parsed UPRN lookup") + logger.debug("[PeabodyParser] Successfully parsed UPRN lookup") return PeabodyParser._group_assets_into_properties( assets=assets, location_ref_to_uprn_map=location_ref_to_uprn_map, @@ -55,7 +55,7 @@ class PeabodyParser(Parser): ) if not asset.is_block_level: # Block-level condition surveys are out of scope for now - # until we have a wider think on how to handle block + # until we have a wider think on how to handle blocks assets.append(asset) # TODO: handle block-level assets except Exception as e: @@ -80,13 +80,14 @@ class PeabodyParser(Parser): assets_by_location_reference[asset.lo_reference].append(asset) properties: List[PeabodyProperty] = [] + failed_mappings_count = 0 for location_ref, grouped_assets in assets_by_location_reference.items(): uprn = location_ref_to_uprn_map.get(location_ref) if uprn is None: - logger.warning(f"No UPRN found for Location Reference: {location_ref}") + failed_mappings_count += 1 continue properties.append( @@ -96,6 +97,7 @@ class PeabodyParser(Parser): ) ) + logger.warning(f"No UPRN found for {failed_mappings_count} Location References") return properties @staticmethod diff --git a/backend/condition/processor.py b/backend/condition/processor.py index b7c7d2b6..ad5b4232 100644 --- a/backend/condition/processor.py +++ b/backend/condition/processor.py @@ -19,10 +19,13 @@ def process_file( uprn_lookup: Optional[UprnLookup], ) -> None: # Instantiation + logger.debug(f"[processor] Instantiating classes...") parser: Parser = select_parser(file_type, uprn_lookup) mapper: Mapper = select_mapper(file_type) persistence = ConditionPostgres() + logger.debug(f"[processor] Finished instantiating classes. Calling Parser...") + # Orchestration raw_properties: List[Any] = parser.parse(file_stream) From d4b444f49bff106efcfbf5e1f7a72693f6ea82d2 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:43:18 +0000 Subject: [PATCH 47/50] Fix bad log --- backend/condition/persistence/condition_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/condition/persistence/condition_postgres.py b/backend/condition/persistence/condition_postgres.py index 9b9ce2e0..bb6ada44 100644 --- a/backend/condition/persistence/condition_postgres.py +++ b/backend/condition/persistence/condition_postgres.py @@ -31,7 +31,7 @@ class ConditionPostgres: ) with db_session() as session: - logger.info("Successfully made connection to database:", session) + logger.info("[ConditionPostgres] Successfully made connection to database") # for start in range(0, total, batch_size): # end = min(start + batch_size, total) # batch = survey_models[start:end] From da34b99251ccb0f7b682e342bf11bc15e7cddcd4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:43:55 +0000 Subject: [PATCH 48/50] tidy up logs further --- backend/condition/persistence/condition_postgres.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/condition/persistence/condition_postgres.py b/backend/condition/persistence/condition_postgres.py index bb6ada44..9428eec6 100644 --- a/backend/condition/persistence/condition_postgres.py +++ b/backend/condition/persistence/condition_postgres.py @@ -19,15 +19,15 @@ class ConditionPostgres: def bulk_insert_surveys( self, surveys: List[PropertyConditionSurvey], batch_size: Optional[int] = 100 ) -> None: - logger.info( - f"Preparing to load {len(surveys)} property surveys to Postgres. Mapping to SQLModel objects..." + logger.debug( + f"[ConditionPostgres] Preparing to load {len(surveys)} property surveys to Postgres. Mapping to SQLModel objects..." ) survey_models: List[PropertyConditionSurveyModel] = [ ConditionPostgres.map_survey_to_model(s) for s in surveys ] total: int = len(survey_models) - logger.info( - f"Finished mapping {total} surveys. Writing to database in batches of {batch_size}..." + logger.debug( + f"[ConditionPostgres] Finished mapping {total} surveys. Writing to database in batches of {batch_size}..." ) with db_session() as session: From df6c42933afb78c7e1f96116aadabf3cfdee6894 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 09:59:23 +0000 Subject: [PATCH 49/50] do the database write --- .../persistence/condition_postgres.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/condition/persistence/condition_postgres.py b/backend/condition/persistence/condition_postgres.py index 9428eec6..e83df540 100644 --- a/backend/condition/persistence/condition_postgres.py +++ b/backend/condition/persistence/condition_postgres.py @@ -32,17 +32,17 @@ class ConditionPostgres: with db_session() as session: logger.info("[ConditionPostgres] Successfully made connection to database") - # for start in range(0, total, batch_size): - # end = min(start + batch_size, total) - # batch = survey_models[start:end] + for start in range(0, total, batch_size): + end = min(start + batch_size, total) + batch = survey_models[start:end] - # t0: float = time.perf_counter() - # ConditionPostgres._insert_surveys_batch(batch, session) - # elapsed: float = time.perf_counter() - t0 + t0: float = time.perf_counter() + ConditionPostgres._insert_surveys_batch(batch, session) + elapsed: float = time.perf_counter() - t0 - # logger.info( - # f"Inserted batch {start} - {end} ({len(batch)} surveys) in {elapsed} seconds", - # ) + logger.info( + f"Inserted batch {start} - {end} ({len(batch)} surveys) in {elapsed} seconds", + ) @staticmethod def map_survey_to_model( From 549eef19dbfc2007ed17b513b396f620a1b9cd32 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 10 Feb 2026 11:13:31 +0000 Subject: [PATCH 50/50] remove token from dockerfile, and add example json payloads --- backend/address2UPRN/handler/Dockerfile | 3 --- backend/condition/condition_trigger_request.py | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/address2UPRN/handler/Dockerfile b/backend/address2UPRN/handler/Dockerfile index 3f7567d3..5a09bd44 100644 --- a/backend/address2UPRN/handler/Dockerfile +++ b/backend/address2UPRN/handler/Dockerfile @@ -1,8 +1,5 @@ FROM public.ecr.aws/lambda/python:3.10 -# This is not going to be permenant - but until we solve for env variables in live prod -ENV EPC_AUTH_TOKEN=a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzAg - # Set working directory (Lambda task root) WORKDIR /var/task diff --git a/backend/condition/condition_trigger_request.py b/backend/condition/condition_trigger_request.py index 0477d044..03bd6ad1 100644 --- a/backend/condition/condition_trigger_request.py +++ b/backend/condition/condition_trigger_request.py @@ -25,3 +25,9 @@ class ConditionTriggerRequest(BaseModel): # "uprn_lookup_file_bucket": "condition-data-dev", # "uprn_lookup_file_key": "input/peabody/uprn-lookup/PeabodyPropertymatched_Dec25_propref_UPRN.csv" # } + +# { +# "file_type": "LBWF", +# "trigger_file_bucket": "condition-data-dev", +# "trigger_file_key": "input/lbwf/LBWF - Example Asset Data September 2025.xlsx", +# }