From ec1df5eaca0f126994834585b05722a29ace2d54 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 15:51:11 +0000 Subject: [PATCH 1/9] modules file changes --- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index 3af33ad7..a62996a1 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -14,7 +14,7 @@ locals { } module "ordnance" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = ordnanceSurvey #"address2uprn" for example stage = var.stage From b9105cc6582688f613e0c2e5798723f169b19e66 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 16:48:37 +0000 Subject: [PATCH 2/9] added ordance survey api --- .github/workflows/_deploy_lambda.yml | 4 ++++ .github/workflows/deploy_terraform.yml | 1 + infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- infrastructure/terraform/lambda/ordnanceSurvey/variables.tf | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index ce1a0e77..103dee82 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -58,6 +58,8 @@ on: required: false TF_VAR_google_solar_api_key: required: false + TF_VAR_ORDNANCE_SURVEY_API_KEY: + required: false jobs: deploy: @@ -115,6 +117,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY}} run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ @@ -140,6 +143,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY}} run: | terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index bde7eb21..6633eefd 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -320,3 +320,4 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index a62996a1..7e045621 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -33,7 +33,7 @@ module "ordnance" { DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - ORDNANCE_SURVEY_API_KEY:= "Reminder to add This somehow, ask if we are doing aws secret method or github secret method" + ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_key }, ) } diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index e7646811..936aebc9 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -28,6 +28,12 @@ variable "batch_size" { default = 1 } +variable "ordnance_survey_api_key" { + type = string + sensitive = true +} + + locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" } From cc1c7d05cb10f0f981e56d7b0f64bd309bfcb753 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 17:00:43 +0000 Subject: [PATCH 3/9] capital letter --- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- infrastructure/terraform/lambda/ordnanceSurvey/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index 7e045621..67732b67 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -33,7 +33,7 @@ module "ordnance" { DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_key + ORDNANCE_SURVEY_API_KEY = var.ORDNANCE_SURVEY_API_KEY }, ) } diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index 936aebc9..6ce3dfd5 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -28,7 +28,7 @@ variable "batch_size" { default = 1 } -variable "ordnance_survey_api_key" { +variable "ORDNANCE_SURVEY_API_KEY" { type = string sensitive = true } From e64bae74ed7d318932fe1960f61e5642d4d0ed78 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 17:07:22 +0000 Subject: [PATCH 4/9] lower case letter --- .github/workflows/_deploy_lambda.yml | 4 ++-- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- infrastructure/terraform/lambda/ordnanceSurvey/variables.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 103dee82..35c517ad 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -117,7 +117,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY}} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ @@ -143,7 +143,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY}} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index 67732b67..b2011d3d 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -33,7 +33,7 @@ module "ordnance" { DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - ORDNANCE_SURVEY_API_KEY = var.ORDNANCE_SURVEY_API_KEY + ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_keY }, ) } diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index 6ce3dfd5..936aebc9 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -28,7 +28,7 @@ variable "batch_size" { default = 1 } -variable "ORDNANCE_SURVEY_API_KEY" { +variable "ordnance_survey_api_key" { type = string sensitive = true } From 48a728721d263da1de832b3964586094c09b6657 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 17:08:14 +0000 Subject: [PATCH 5/9] lower case --- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index b2011d3d..7e045621 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -33,7 +33,7 @@ module "ordnance" { DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_keY + ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_key }, ) } From 6642e1b357f43eec38922802cebc707ba034648c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 17:09:43 +0000 Subject: [PATCH 6/9] lower case --- .github/workflows/_deploy_lambda.yml | 2 +- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 35c517ad..6d1430a2 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -58,7 +58,7 @@ on: required: false TF_VAR_google_solar_api_key: required: false - TF_VAR_ORDNANCE_SURVEY_API_KEY: + TF_VAR_ordnance_survey_api_key: required: false jobs: diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index 7e045621..fb8876fa 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -42,4 +42,4 @@ module "ordnance" { resource "aws_iam_role_policy_attachment" "ordanceSurvey_read_and_write" { role = module.ordnance.role_name policy_arn = data.terraform_remote_state.shared.outputs.ordnance_s3_read_and_write_arn -} +} \ No newline at end of file From 83ce83321e814cfd9525fd283db5fdae450b0d11 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 9 Mar 2026 17:17:01 +0000 Subject: [PATCH 7/9] lower case --- infrastructure/terraform/lambda/ordnanceSurvey/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index fb8876fa..ace03ffc 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -16,7 +16,7 @@ locals { module "ordnance" { source = "../../modules/lambda_with_sqs" - name = ordnanceSurvey #"address2uprn" for example + name = "ordnanceSurvey" #"address2uprn" for example stage = var.stage image_uri = local.image_uri From 5a9881552928880a5e925798833ef1f948dc6d10 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 10 Mar 2026 10:10:59 +0000 Subject: [PATCH 8/9] didn't deploy --- .github/workflows/deploy_terraform.yml | 1 + backend/address2UPRN/README.md | 30 +++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 6633eefd..5841030c 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -316,6 +316,7 @@ jobs: stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/backend/address2UPRN/README.md b/backend/address2UPRN/README.md index b4876340..390aeb62 100644 --- a/backend/address2UPRN/README.md +++ b/backend/address2UPRN/README.md @@ -1,20 +1,26 @@ -We have list of address as input. - -It'll come in batches of the same post code and from then we want to somehow convert that into UPRN - -if this lambda/function can do that we'll be speeding ahead +So you want to fetch UPRN for an address list? -Energy Performance Information: https://epc.opendatacommunities.org/ +Before you run: -guidance page: https://epc.opendatacommunities.org/docs/guidance#field_domestic_LMK_KEY +Step 1) Get the list and ensure the following columns exists -Example of past khalims code that he wrote some tests for: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/tests/test_search_epc.py#L11 +* Address 1 +* Address 2 +* Address 3 +* postcode + +And save it as a .csv file -Example of EPC search: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/SearchEpc.py#L118 +Step 2) + +Before we run this, we need to upload it into S3 as well as put initiate a subtask + task + +* S3 upload I'll recommend somewhere in retrofit-data-dev and get the s3_uri + +For this example I'll be using "s3://retrofit-data-dev/ara_raw_inputs/calico/formated(Sheet1).csv" + +Go to Ara DB and make a new task_id with a randomly generated uuid as the primarily key - -Khalim has made a python package to help scrape data: https://github.com/KhalimCK/epc-api-python - From d42ba5b9cf6e47f22512c9e84a66ca00bd6fd3a2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 10 Mar 2026 10:55:57 +0000 Subject: [PATCH 9/9] ecr repo --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 5841030c..596ab82d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -314,7 +314,7 @@ jobs: lambda_name: ordnanceSurvey lambda_path: infrastructure/terraform/lambda/ordnanceSurvey stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} + ecr_repo: ordnance-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }} terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: