Merge branch 'main' into feature/categorisation-improvements

.github/workflows/unit_tests.yml

@@ -1,11 +1,6 @@
 name: Run unit tests
 
 on:
-  push:
-    branches:
-      - "main"
-      - "dev"
-      - "prod"
   pull_request:
     branches:
       - "**"

asset_list/app.py

@@ -18,6 +18,7 @@ EPC_AUTH_TOKEN = os.getenv(
     "EPC_AUTH_TOKEN",
 )
 
+
 OPENAI_API_KEY = os.getenv(
     "OPENAI_API_KEY",
 )
@@ -73,61 +74,24 @@ def app():
     Property UPRN
     """
 
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/West Kent"
-    data_filename = "West Kent Asset List.xlsx"
+    data_folder = "/workspaces/model/asset_list"
+    data_filename = "assests.xlsx"
     sheet_name = "Sheet1"
-    postcode_column = "POSTCODE"
-    address1_column = None
+    postcode_column = "Postcode"
+    address1_column = "Address"
     address1_method = "house_number_extraction"
-    fulladdress_column = "ADDRESS"
-    address_cols_to_concat = []
+    fulladdress_column = None
+    address_cols_to_concat = ["Address"]
     missing_postcodes_method = None
     landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = "PROPERTY TYPE"
-    landlord_built_form = None
-    landlord_wall_construction = "wall combined"
-    landlord_roof_construction = "HEATING SYSTEM"
-    landlord_heating_system = None
+    landlord_os_uprn = "UPRN"
+    landlord_property_type = "Archetype"
+    landlord_built_form = "Bedroom Count"
+    landlord_wall_construction = "Wall Insulation Type"
+    landlord_roof_construction = "Roof Type"
+    landlord_heating_system = "Boiler Type"
     landlord_existing_pv = None
-    landlord_property_id = "UPRN"
-    landlord_sap = None
-    outcomes_filename = None
-    outcomes_sheetname = None
-    outcomes_postcode = None
-    outcomes_houseno = None
-    outcomes_id = None
-    outcomes_address = None
-    master_filepaths = []
-    master_id_colnames = []
-    master_to_asset_list_filepath = None
-    phase = False
-    ecosurv_landlords = None
-    asset_list_header = 0
-    landlord_block_reference = None
-
-    # Peabody data for cleaning
-    data_folder = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting "
-        "Project/data_validation"
-    )
-    data_filename = "to_standardise_uprns.xlsx"
-    sheet_name = "Sheet1"
-    postcode_column = "POSTCODE"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    fulladdress_column = "ADDRESS"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = "PROPERTY TYPE"
-    landlord_built_form = None  # Skipped as empty
-    landlord_wall_construction = "wall combined"  # combin F + G
-    landlord_roof_construction = "HEATING SYSTEM"  # Combine I + J
-    landlord_heating_system = None  # Check with Khalim
-    landlord_existing_pv = None
-    landlord_property_id = "UPRN"
+    landlord_property_id = "Tab"
     landlord_sap = None
     outcomes_filename = None
     outcomes_sheetname = None

asset_list/requirements.txt

@@ -5,7 +5,7 @@ epc-api-python==1.0.2
 thefuzz
 boto3
 openpyxl
-openai>=1.3.5
+openai==1.93.0
 tiktoken
 msgpack
 beautifulsoup4

infrastructure/terraform/lambda/_template/main.tf

@@ -33,6 +33,8 @@ module "lambda" {
 
   image_uri = local.image_uri
 
+  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
+  maximum_concurrency = var.maximum_concurrency
 
   environment = {
     STAGE = var.stage

infrastructure/terraform/lambda/_template/variables.tf

@@ -17,6 +17,11 @@ variable "image_digest" {
   description = "Image digest (sha256:...)"
 }
 
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
+}
 
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"

infrastructure/terraform/lambda/address2UPRN/main.tf

@@ -24,6 +24,9 @@ module "address2uprn" {
 
   timeout = 900
 
+  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
+  maximum_concurrency = var.maximum_concurrency
+
   environment = merge(
     {
       STAGE     = var.stage

infrastructure/terraform/lambda/address2UPRN/variables.tf

@@ -17,6 +17,11 @@ variable "image_digest" {
   description = "Image digest (sha256:...)"
 }
 
+variable "maximum_concurrency" {
+  type        = number
+  default     = 10 # null if you don't want to set it for this handler
+  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
+}
 
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"

infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf

@@ -44,5 +44,6 @@ module "sqs_trigger" {
   lambda_role_name = module.role.role_name
   queue_arn        = module.queue.queue_arn
 
-  batch_size = var.batch_size
+  batch_size           = var.batch_size
+  maximum_concurrency  = var.maximum_concurrency
 }

infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf

@@ -34,3 +34,9 @@ variable "batch_size" {
   type    = number
   default = 10
 }
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit."
+}

infrastructure/terraform/modules/lambda_sqs_trigger/main.tf

@@ -3,6 +3,13 @@ resource "aws_lambda_event_source_mapping" "this" {
   function_name    = var.lambda_arn
   batch_size       = var.batch_size
   enabled          = true
+
+  dynamic "scaling_config" {
+    for_each = var.maximum_concurrency != null ? [1] : []
+    content {
+      maximum_concurrency = var.maximum_concurrency
+    }
+  }
 }
 
 resource "aws_iam_role_policy" "allow_sqs" {

infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf

@@ -6,3 +6,9 @@ variable "batch_size" {
   type    = number
   default = 10
 }
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit."
+}