diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 7e11cf32..420f9504 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -60,6 +60,8 @@ on: required: false TF_VAR_google_solar_api_key: required: false + TF_VAR_ordnance_survey_api_key: + required: false jobs: deploy: @@ -117,6 +119,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | EXTRA_VARS="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then @@ -146,6 +149,7 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | EXTRA_VARS="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 042bce35..366cd004 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -314,9 +314,11 @@ jobs: lambda_name: ordnanceSurvey lambda_path: infrastructure/terraform/lambda/ordnanceSurvey stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} + ecr_repo: ordnance-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} diff --git a/backend/address2UPRN/README.md b/backend/address2UPRN/README.md index b4876340..390aeb62 100644 --- a/backend/address2UPRN/README.md +++ b/backend/address2UPRN/README.md @@ -1,20 +1,26 @@ -We have list of address as input. - -It'll come in batches of the same post code and from then we want to somehow convert that into UPRN - -if this lambda/function can do that we'll be speeding ahead +So you want to fetch UPRN for an address list? -Energy Performance Information: https://epc.opendatacommunities.org/ +Before you run: -guidance page: https://epc.opendatacommunities.org/docs/guidance#field_domestic_LMK_KEY +Step 1) Get the list and ensure the following columns exists -Example of past khalims code that he wrote some tests for: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/tests/test_search_epc.py#L11 +* Address 1 +* Address 2 +* Address 3 +* postcode + +And save it as a .csv file -Example of EPC search: https://github.com/Hestia-Homes/Model/blob/941be42b83a590e838fd3ee475bfd1ff31438789/backend/SearchEpc.py#L118 +Step 2) + +Before we run this, we need to upload it into S3 as well as put initiate a subtask + task + +* S3 upload I'll recommend somewhere in retrofit-data-dev and get the s3_uri + +For this example I'll be using "s3://retrofit-data-dev/ara_raw_inputs/calico/formated(Sheet1).csv" + +Go to Ara DB and make a new task_id with a randomly generated uuid as the primarily key - -Khalim has made a python package to help scrape data: https://github.com/KhalimCK/epc-api-python - diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf index 3af33ad7..ace03ffc 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/main.tf @@ -14,9 +14,9 @@ locals { } module "ordnance" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" - name = ordnanceSurvey #"address2uprn" for example + name = "ordnanceSurvey" #"address2uprn" for example stage = var.stage image_uri = local.image_uri @@ -33,7 +33,7 @@ module "ordnance" { DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - ORDNANCE_SURVEY_API_KEY:= "Reminder to add This somehow, ask if we are doing aws secret method or github secret method" + ORDNANCE_SURVEY_API_KEY = var.ordnance_survey_api_key }, ) } @@ -42,4 +42,4 @@ module "ordnance" { resource "aws_iam_role_policy_attachment" "ordanceSurvey_read_and_write" { role = module.ordnance.role_name policy_arn = data.terraform_remote_state.shared.outputs.ordnance_s3_read_and_write_arn -} +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index e7646811..936aebc9 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -28,6 +28,12 @@ variable "batch_size" { default = 1 } +variable "ordnance_survey_api_key" { + type = string + sensitive = true +} + + locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" }