Merge branch 'main' into deploy-fastapi-with-terraform

.github/workflows/_deploy_lambda.yml

@@ -60,6 +60,8 @@ on:
         required: false
       TF_VAR_google_solar_api_key:
         required: false
+      TF_VAR_ordnance_survey_api_key:
+        required: false
 
 jobs:
   deploy:
@@ -117,6 +119,7 @@ jobs:
           TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
           TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
           TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
+          TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}}
         run: |
           EXTRA_VARS=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then
@@ -146,6 +149,7 @@ jobs:
           TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
           TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
           TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
+          TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}}
         run: |
           EXTRA_VARS=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then

.github/workflows/deploy_terraform.yml

@@ -314,9 +314,11 @@ jobs:
       lambda_name: ordnanceSurvey
       lambda_path: infrastructure/terraform/lambda/ordnanceSurvey
       stage: ${{ needs.determine_stage.outputs.stage }}
-      ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }}
+      ecr_repo: ordnance-${{ needs.determine_stage.outputs.stage }}
       image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }}
+      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+      TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }}

backend/address2UPRN/README.md

@@ -1,20 +1,26 @@
-We have list of address as input.
-
-It'll come in batches of the same post code and from then we want to somehow convert that into UPRN
-
-if this lambda/function can do that we'll be speeding ahead
+So you want to fetch UPRN for an address list?
 
 
-Energy Performance Information: https://epc.opendatacommunities.org/
+Before you run:
 
-guidance page: https://epc.opendatacommunities.org/docs/guidance#field_domestic_LMK_KEY
+Step 1) Get the list and ensure the following columns exists
 
-Example of past khalims code that he wrote some tests for: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/tests/test_search_epc.py#L11
+* Address 1
+* Address 2
+* Address 3
+* postcode
+
+And save it as a .csv file
 
 
-Example of EPC search: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/SearchEpc.py#L118
+Step 2)
+
+Before we run this, we need to upload it into S3 as well as put initiate a subtask + task
+
+* S3 upload I'll recommend somewhere in retrofit-data-dev and get the s3_uri
+
+For this example I'll be using "s3://retrofit-data-dev/ara_raw_inputs/calico/formated(Sheet1).csv"
+
+Go to Ara DB and make a new task_id with a randomly generated uuid as the primarily key
 
 
-
-Khalim has made a python package to help scrape data: https://github.com/KhalimCK/epc-api-python
-

infrastructure/terraform/lambda/ordnanceSurvey/main.tf

@@ -14,9 +14,9 @@ locals {
 }
 
 module "ordnance" {
-  source = "../modules/lambda_with_sqs"
+  source = "../../modules/lambda_with_sqs"
 
-  name  = ordnanceSurvey #"address2uprn" for example
+  name  = "ordnanceSurvey" #"address2uprn" for example
   stage = var.stage
 
   image_uri = local.image_uri
@@ -33,7 +33,7 @@ module "ordnance" {
       DB_USERNAME = local.db_credentials.db_assessment_model_username
       DB_PASSWORD = local.db_credentials.db_assessment_model_password
       S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
-      ORDNANCE_SURVEY_API_KEY:= "Reminder to add This somehow, ask if we are doing aws secret method or github secret method"
+      ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_key
     },
   )
 }
@@ -42,4 +42,4 @@ module "ordnance" {
 resource "aws_iam_role_policy_attachment" "ordanceSurvey_read_and_write" {
   role       = module.ordnance.role_name
   policy_arn = data.terraform_remote_state.shared.outputs.ordnance_s3_read_and_write_arn
-}
+}

infrastructure/terraform/lambda/ordnanceSurvey/variables.tf

@@ -28,6 +28,12 @@ variable "batch_size" {
   default = 1
 }
 
+variable "ordnance_survey_api_key" {
+  type      = string
+  sensitive = true
+}
+
+
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"
 }