From 49892a8f60e266ff68da44c95594c10defe116b6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 10 Mar 2026 16:15:02 +0000 Subject: [PATCH 1/7] updated readme --- backend/address2UPRN/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/address2UPRN/README.md b/backend/address2UPRN/README.md index 1a835b6e..6d26f281 100644 --- a/backend/address2UPRN/README.md +++ b/backend/address2UPRN/README.md @@ -53,3 +53,6 @@ ordnance_survey sqs is => https://eu-west-2.console.aws.amazon.com/sqs/v3/home?r "task_id": "a7b70a02-4df4-45b5-a50b-196e095910bb", "sub_task_id": "567cf73b-1210-4909-9ecc-36ae7e23420e" } + + +outputs are at s3://retrofit-data-dev/ara_ordnance_survey_outputs// \ No newline at end of file From 73cd47def1702ef1c5b828843920e4ee883c9e6d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 11 Mar 2026 15:04:46 +0000 Subject: [PATCH 2/7] cotality script --- cotality.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 cotality.py diff --git a/cotality.py b/cotality.py new file mode 100644 index 00000000..43f9afea --- /dev/null +++ b/cotality.py @@ -0,0 +1,73 @@ +import requests +import json + +TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik1EUTRNRU5GUTBVNU9FUXpOelk1TVRFME0wUkdOMFpFUkRoR1JVVkJNVGMxT1RFNFJERXlPQSJ9.eyJodHRwOi8vZW1haWwiOiJzZWJhc3RpYW5Ab3Ntb3Npcy1hY2QuY29tIiwiaHR0cDovL2NsdWsudG9rZW4vbGFzdFBhc3N3b3JkQ2hhbmdlIjoiMjAyNS0wOC0yNlQwOTo1NDoyNi4zMjZaIiwiaHR0cDovL2NsdWsudG9rZW4vY29ubmVjdGlvbiI6ImVUZWNoSUQiLCJodHRwOi8vY2x1ay50b2tlbi9zdHJhdGVneSI6ImF1dGgwIiwiaHR0cDovL2NsdWsudG9rZW4vc3RyYXRlZ3lUeXBlIjoiZGF0YWJhc2UiLCJpc3MiOiJodHRwczovL2V0ZWNoaWQuZXUuYXV0aDAuY29tLyIsInN1YiI6ImF1dGgwfDY4YWQ4NDUyZDI2YzI1ZmMyMzkwZmYxYSIsImF1ZCI6WyJodHRwczovL3Bhc2h1Yi5hcGkuZXRlY2gubmV0IiwiaHR0cHM6Ly9ldGVjaGlkLmV1LmF1dGgwLmNvbS91c2VyaW5mbyJdLCJpYXQiOjE3NzMyMzc4MjQsImV4cCI6MTc3MzI0NTAyNCwic2NvcGUiOiJvcGVuaWQiLCJhenAiOiJEaVp6d3VVaTVkVmozOXR3NG00bWZ6emZvRm5MdmVLZyJ9.mkkxeZiD_ByHY4TJKpLQ-trmeGs15s0ekL6u1n-ek9j-EzNyf6qalEHCyHf8gzdNhU_vay96bIOMRHp4vXFaLqSANwKZayIS3EoA_b9-u2FAZpooxEvReAMNJGoZ6WLD01AQXWv-l7ww1ZqAnQzw0moL_Oma6hVmA5oa-RJKJ3MerS7e0Wei97Db48E140-EAbQf2iPcKYYtCNRA4il6n8DFiqGeoUMGo99jkR1ceZAvMpOAj8RhKX-4qSiDfX6yXUS2G96U5m7S_GWI-DEj5TazkN10Af3TyOY3EVjmZoJcRpiAR4cFmlfcTydjrShU03DWmPZm1QItf2McxfCpNA" + +base = "https://pashub.net/api" + +headers = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"} + +company_id = "cb5249e2-8f31-4ef4-aefd-08ddaccb1fa2" + +# 1️⃣ get jobs +params = { + "pageIndex": 0, + "pageSize": 20, + "orderBy": "createdUtc", + "orderDesc": "true", + "addressUprn": "100061885568", + "companyId": company_id, +} + +r = requests.get(f"{base}/jobs", headers=headers, params=params) + +payload = r.json() + +property_id = payload["results"][0]["id"] + +print("JOB:", property_id) + +# 2️⃣ get evidence list +r = requests.get(f"{base}/jobs/{property_id}/evidence", headers=headers) + +print(r.status_code) + +evidence = r.json() + +print(evidence) + + +# 3️⃣ get evidence metadata + +if evidence: + evidence_id = evidence["results"][0]["fileId"] + + meta_url = f"https://pashub.net/api/jobs/{property_id}/evidenceMetadata" + + meta_params = {"evidenceIds": evidence_id} + + r = requests.get(meta_url, headers=headers, params=meta_params) + r.raise_for_status() + + meta = r.json() + + container = meta["containerName"] + blob_uri = meta["blobUri"] + + file = meta["files"][0] + file_id = file["fileId"] + file_name = file["fileName"] + + base, sas = blob_uri.split("?", 1) + + download_url = f"{base}{container}/{file_id}?{sas}" + + print("Download URL:", download_url) + + pdf = requests.get(download_url) + pdf.raise_for_status() + + with open(file_name, "wb") as f: + f.write(pdf.content) + + print("Saved:", file_name) From dde17250be83bc9bcea7af18f034e3ec7edcc593 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 11 Mar 2026 15:07:55 +0000 Subject: [PATCH 3/7] renamed --- cotality.py => download_cotality_evidence.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cotality.py => download_cotality_evidence.py (100%) diff --git a/cotality.py b/download_cotality_evidence.py similarity index 100% rename from cotality.py rename to download_cotality_evidence.py From f102aa6a7c241172a8947949a0687465246d59cd Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 11 Mar 2026 15:27:31 +0000 Subject: [PATCH 4/7] move location --- .../download_cotality_evidence.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename download_cotality_evidence.py => scripts/download_cotality_evidence.py (100%) diff --git a/download_cotality_evidence.py b/scripts/download_cotality_evidence.py similarity index 100% rename from download_cotality_evidence.py rename to scripts/download_cotality_evidence.py From 5a6294b79e95c8fa05c5ba29681ec3a4b37f03e9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 11 Mar 2026 17:45:15 +0000 Subject: [PATCH 5/7] exclude tests from lambda zip --- .../terraform/modules/lambda_with_api_gateway/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf index b32380de..ba2d844e 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf @@ -6,7 +6,7 @@ variable "runtime" { type = string } variable "zip_excludes" { type = list(string) - default = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**"] + default = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] } variable "timeout" { From c58cfee0b631eb699b3ebda5dc5eb3c191473991 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 12 Mar 2026 10:41:08 +0000 Subject: [PATCH 6/7] upload zip to s3 rather than copying via api --- infrastructure/terraform/lambda/fast-api/main.tf | 1 + .../terraform/modules/lambda_service_zip/main.tf | 8 ++++++-- .../modules/lambda_service_zip/variables.tf | 7 +++++-- .../modules/lambda_with_api_gateway/main.tf | 13 ++++++++++++- .../modules/lambda_with_api_gateway/variables.tf | 3 ++- infrastructure/terraform/shared/main.tf | 6 ------ 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index c9058fb7..9e8c7c2b 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -82,6 +82,7 @@ module "fastapi" { runtime = "python3.11" timeout = 600 memory_size = 512 + artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket # domain_name = "api.${var.domain_name}" # certificate_arn = data.aws_ssm_parameter.certificate_arn.value diff --git a/infrastructure/terraform/modules/lambda_service_zip/main.tf b/infrastructure/terraform/modules/lambda_service_zip/main.tf index 285aa9d4..232b5b56 100644 --- a/infrastructure/terraform/modules/lambda_service_zip/main.tf +++ b/infrastructure/terraform/modules/lambda_service_zip/main.tf @@ -2,7 +2,7 @@ resource "aws_lambda_function" "this" { function_name = var.name role = var.role_arn package_type = "Zip" - filename = var.filename + # filename = var.filename source_code_hash = var.source_code_hash handler = var.handler runtime = var.runtime @@ -10,6 +10,10 @@ resource "aws_lambda_function" "this" { memory_size = var.memory_size publish = true + s3_bucket = var.s3_bucket + s3_key = var.s3_key + source_code_hash = var.source_code_hash + environment { variables = var.environment } @@ -21,4 +25,4 @@ output "lambda_arn" { output "function_name" { value = aws_lambda_function.this.function_name -} \ No newline at end of file +} diff --git a/infrastructure/terraform/modules/lambda_service_zip/variables.tf b/infrastructure/terraform/modules/lambda_service_zip/variables.tf index 68a35370..85d1f548 100644 --- a/infrastructure/terraform/modules/lambda_service_zip/variables.tf +++ b/infrastructure/terraform/modules/lambda_service_zip/variables.tf @@ -1,6 +1,6 @@ variable "name" { type = string } variable "role_arn" { type = string } -variable "filename" { type = string } +# variable "filename" { type = string } variable "source_code_hash" { type = string } variable "handler" { type = string } variable "runtime" { type = string } @@ -15,4 +15,7 @@ variable "memory_size" { variable "environment" { type = map(string) default = {} -} \ No newline at end of file +} +variable "s3_bucket" { type = string } +variable "s3_key" { type = string } +variable "source_code_hash" { type = string } \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf index 61e24c32..2277dee5 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf @@ -16,6 +16,16 @@ data "archive_file" "this" { excludes = var.zip_excludes } +############################################ +# Upload zip to S3 +############################################ +resource "aws_s3_object" "lambda_zip" { + bucket = var.artifact_bucket + key = "${var.name}-${var.stage}.zip" + source = data.archive_file.this.output_path + etag = data.archive_file.this.output_md5 +} + ############################################ # Lambda ############################################ @@ -24,7 +34,8 @@ module "lambda" { name = "${var.name}-${var.stage}" role_arn = module.role.role_arn - filename = data.archive_file.this.output_path + s3_bucket = var.artifact_bucket + s3_key = aws_s3_object.lambda_zip.key source_code_hash = data.archive_file.this.output_base64sha256 handler = var.handler runtime = var.runtime diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf index ba2d844e..0b1dfe71 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf @@ -33,4 +33,5 @@ variable "certificate_arn" { variable "route53_zone_id" { type = string default = null -} \ No newline at end of file +} +variable "artifact_bucket" { type = string } \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 1e88435d..8a4e4a1f 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -530,12 +530,6 @@ module "ara_fast_api_state_bucket" { bucket_name = "ara-fast-api-terraform-state" } -module "ara_fastapi_registry" { - source = "../modules/container_registry" - name = "ara-fastapi" - stage = var.stage -} - # S3 policy for FastAPI app to read and write from various S3 buckets module "fast_api_s3_read_and_write" { source = "../modules/s3_iam_policy" From 597fb2e764abe7814e705efa105ce1e9d6fd96b5 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 12 Mar 2026 10:47:27 +0000 Subject: [PATCH 7/7] modify key to match tfstate path --- .../terraform/modules/lambda_with_api_gateway/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf index 2277dee5..f72729e4 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf @@ -21,7 +21,7 @@ data "archive_file" "this" { ############################################ resource "aws_s3_object" "lambda_zip" { bucket = var.artifact_bucket - key = "${var.name}-${var.stage}.zip" + key = "env:/${var.stage}/${var.name}.zip" source = data.archive_file.this.output_path etag = data.archive_file.this.output_md5 }