Merge pull request #850 from Hestia-Homes/main

Deploy fastapi with terraform: exclude tests from zip file

backend/address2UPRN/README.md

@@ -53,3 +53,6 @@ ordnance_survey sqs is => https://eu-west-2.console.aws.amazon.com/sqs/v3/home?r
     "task_id": "a7b70a02-4df4-45b5-a50b-196e095910bb",
     "sub_task_id": "567cf73b-1210-4909-9ecc-36ae7e23420e"
 }
+
+
+outputs are at s3://retrofit-data-dev/ara_ordnance_survey_outputs/<task-id>/<sub-task-id>

infrastructure/terraform/lambda/fast-api/main.tf

@@ -82,6 +82,7 @@ module "fastapi" {
   runtime     = "python3.11"
   timeout     = 600
   memory_size = 512
+  artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
 
   # domain_name     = "api.${var.domain_name}"
   # certificate_arn = data.aws_ssm_parameter.certificate_arn.value

infrastructure/terraform/modules/lambda_service_zip/main.tf

@@ -2,7 +2,7 @@ resource "aws_lambda_function" "this" {
   function_name    = var.name
   role             = var.role_arn
   package_type     = "Zip"
-  filename         = var.filename
+  # filename         = var.filename
   source_code_hash = var.source_code_hash
   handler          = var.handler
   runtime          = var.runtime
@@ -10,6 +10,10 @@ resource "aws_lambda_function" "this" {
   memory_size      = var.memory_size
   publish          = true
 
+  s3_bucket        = var.s3_bucket
+  s3_key           = var.s3_key
+  source_code_hash = var.source_code_hash
+
   environment {
     variables = var.environment
   }
@@ -21,4 +25,4 @@ output "lambda_arn" {
 
 output "function_name" {
   value = aws_lambda_function.this.function_name
-}
+}

infrastructure/terraform/modules/lambda_service_zip/variables.tf

@@ -1,6 +1,6 @@
 variable "name"             { type = string }
 variable "role_arn"         { type = string }
-variable "filename"         { type = string }
+# variable "filename"         { type = string }
 variable "source_code_hash" { type = string }
 variable "handler"          { type = string }
 variable "runtime"          { type = string }
@@ -15,4 +15,7 @@ variable "memory_size"      {
 variable "environment"      {
     type = map(string)
     default = {}
-}
+}
+variable "s3_bucket"        { type = string }
+variable "s3_key"           { type = string }
+variable "source_code_hash" { type = string }

infrastructure/terraform/modules/lambda_with_api_gateway/main.tf

@@ -16,6 +16,16 @@ data "archive_file" "this" {
   excludes    = var.zip_excludes
 }
 
+############################################
+# Upload zip to S3
+############################################
+resource "aws_s3_object" "lambda_zip" {
+  bucket = var.artifact_bucket
+  key    = "env:/${var.stage}/${var.name}.zip"
+  source = data.archive_file.this.output_path
+  etag   = data.archive_file.this.output_md5
+}
+
 ############################################
 # Lambda
 ############################################
@@ -24,7 +34,8 @@ module "lambda" {
 
   name             = "${var.name}-${var.stage}"
   role_arn         = module.role.role_arn
-  filename         = data.archive_file.this.output_path
+  s3_bucket        = var.artifact_bucket
+  s3_key           = aws_s3_object.lambda_zip.key
   source_code_hash = data.archive_file.this.output_base64sha256
   handler          = var.handler
   runtime          = var.runtime

infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf

@@ -6,7 +6,7 @@ variable "runtime"    { type = string }
 
 variable "zip_excludes" {
   type    = list(string)
-  default = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**"]
+  default = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"]
 }
 
 variable "timeout"     {
@@ -33,4 +33,5 @@ variable "certificate_arn" {
 variable "route53_zone_id" {
   type = string
   default = null
-}
+}
+variable "artifact_bucket" { type = string }

infrastructure/terraform/shared/main.tf

@@ -530,12 +530,6 @@ module "ara_fast_api_state_bucket" {
   bucket_name = "ara-fast-api-terraform-state"
 }
 
-module "ara_fastapi_registry" {
-  source = "../modules/container_registry"
-  name   = "ara-fastapi"
-  stage = var.stage
-}
-
 # S3 policy for FastAPI app to read and write from various S3 buckets
 module "fast_api_s3_read_and_write" {
   source = "../modules/s3_iam_policy"

scripts/download_cotality_evidence.py

@@ -0,0 +1,73 @@
+import requests
+import json
+
+TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik1EUTRNRU5GUTBVNU9FUXpOelk1TVRFME0wUkdOMFpFUkRoR1JVVkJNVGMxT1RFNFJERXlPQSJ9.eyJodHRwOi8vZW1haWwiOiJzZWJhc3RpYW5Ab3Ntb3Npcy1hY2QuY29tIiwiaHR0cDovL2NsdWsudG9rZW4vbGFzdFBhc3N3b3JkQ2hhbmdlIjoiMjAyNS0wOC0yNlQwOTo1NDoyNi4zMjZaIiwiaHR0cDovL2NsdWsudG9rZW4vY29ubmVjdGlvbiI6ImVUZWNoSUQiLCJodHRwOi8vY2x1ay50b2tlbi9zdHJhdGVneSI6ImF1dGgwIiwiaHR0cDovL2NsdWsudG9rZW4vc3RyYXRlZ3lUeXBlIjoiZGF0YWJhc2UiLCJpc3MiOiJodHRwczovL2V0ZWNoaWQuZXUuYXV0aDAuY29tLyIsInN1YiI6ImF1dGgwfDY4YWQ4NDUyZDI2YzI1ZmMyMzkwZmYxYSIsImF1ZCI6WyJodHRwczovL3Bhc2h1Yi5hcGkuZXRlY2gubmV0IiwiaHR0cHM6Ly9ldGVjaGlkLmV1LmF1dGgwLmNvbS91c2VyaW5mbyJdLCJpYXQiOjE3NzMyMzc4MjQsImV4cCI6MTc3MzI0NTAyNCwic2NvcGUiOiJvcGVuaWQiLCJhenAiOiJEaVp6d3VVaTVkVmozOXR3NG00bWZ6emZvRm5MdmVLZyJ9.mkkxeZiD_ByHY4TJKpLQ-trmeGs15s0ekL6u1n-ek9j-EzNyf6qalEHCyHf8gzdNhU_vay96bIOMRHp4vXFaLqSANwKZayIS3EoA_b9-u2FAZpooxEvReAMNJGoZ6WLD01AQXWv-l7ww1ZqAnQzw0moL_Oma6hVmA5oa-RJKJ3MerS7e0Wei97Db48E140-EAbQf2iPcKYYtCNRA4il6n8DFiqGeoUMGo99jkR1ceZAvMpOAj8RhKX-4qSiDfX6yXUS2G96U5m7S_GWI-DEj5TazkN10Af3TyOY3EVjmZoJcRpiAR4cFmlfcTydjrShU03DWmPZm1QItf2McxfCpNA"
+
+base = "https://pashub.net/api"
+
+headers = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"}
+
+company_id = "cb5249e2-8f31-4ef4-aefd-08ddaccb1fa2"
+
+# 1️⃣ get jobs
+params = {
+    "pageIndex": 0,
+    "pageSize": 20,
+    "orderBy": "createdUtc",
+    "orderDesc": "true",
+    "addressUprn": "100061885568",
+    "companyId": company_id,
+}
+
+r = requests.get(f"{base}/jobs", headers=headers, params=params)
+
+payload = r.json()
+
+property_id = payload["results"][0]["id"]
+
+print("JOB:", property_id)
+
+# 2️⃣ get evidence list
+r = requests.get(f"{base}/jobs/{property_id}/evidence", headers=headers)
+
+print(r.status_code)
+
+evidence = r.json()
+
+print(evidence)
+
+
+# 3️⃣ get evidence metadata
+
+if evidence:
+    evidence_id = evidence["results"][0]["fileId"]
+
+    meta_url = f"https://pashub.net/api/jobs/{property_id}/evidenceMetadata"
+
+    meta_params = {"evidenceIds": evidence_id}
+
+    r = requests.get(meta_url, headers=headers, params=meta_params)
+    r.raise_for_status()
+
+    meta = r.json()
+
+    container = meta["containerName"]
+    blob_uri = meta["blobUri"]
+
+    file = meta["files"][0]
+    file_id = file["fileId"]
+    file_name = file["fileName"]
+
+    base, sas = blob_uri.split("?", 1)
+
+    download_url = f"{base}{container}/{file_id}?{sas}"
+
+    print("Download URL:", download_url)
+
+    pdf = requests.get(download_url)
+    pdf.raise_for_status()
+
+    with open(file_name, "wb") as f:
+        f.write(pdf.content)
+
+    print("Saved:", file_name)