Merge branch 'main' of https://github.com/Hestia-Homes/Model into bug/plan-with-budget-more-expensive

# Conflicts:
#	asset_list/app.py

.github/workflows/unit_tests.yml

@@ -1,11 +1,6 @@
 name: Run unit tests
 
 on:
-  push:
-    branches:
-      - "main"
-      - "dev"
-      - "prod"
   pull_request:
     branches:
       - "**"

asset_list/app.py

@@ -18,6 +18,7 @@ EPC_AUTH_TOKEN = os.getenv(
     "EPC_AUTH_TOKEN",
 )
 
+
 OPENAI_API_KEY = os.getenv(
     "OPENAI_API_KEY",
 )
@@ -73,7 +74,7 @@ def app():
     Property UPRN
     """
 
-    data_folder = "/Users/khalimconn-kowlessar/Downloads"
+    data_folder = "/workspaces/model/asset_list"
     data_filename = "assests.xlsx"
     sheet_name = "Sheet1"
     postcode_column = "Postcode"
@@ -242,7 +243,7 @@ def app():
         if skip is not None and not force_retrieve_data:
             if i <= skip:
                 continue
-        chunk = asset_list.standardised_asset_list[i: i + chunk_size]
+        chunk = asset_list.standardised_asset_list[i : i + chunk_size]
         epc_data_chunk, errors_chunk, no_epc_chunk = get_data(
             df=chunk,
             row_id_name=asset_list.DOMNA_PROPERTY_ID,
@@ -385,7 +386,7 @@ def app():
     # Retrieve just the data we need
     epc_df = epc_df[
         [asset_list.DOMNA_PROPERTY_ID] + list(asset_list.EPC_API_DATA_NAMES.keys())
-        ].rename(columns=asset_list.EPC_API_DATA_NAMES)
+    ].rename(columns=asset_list.EPC_API_DATA_NAMES)
 
     # Look for columns not in the find my EPC data, which will have happened if we didn't
     # retrieve it in the first place
@@ -402,7 +403,7 @@ def app():
         find_my_epc_data[
             [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"]
             + list(asset_list.FIND_EPC_DATA_NAMES.keys())
-            ].rename(columns=asset_list.FIND_EPC_DATA_NAMES),
+        ].rename(columns=asset_list.FIND_EPC_DATA_NAMES),
         how="left",
         on=asset_list.DOMNA_PROPERTY_ID,
     )

asset_list/requirements.txt

@@ -5,7 +5,7 @@ epc-api-python==1.0.2
 thefuzz
 boto3
 openpyxl
-openai>=1.3.5
+openai==1.93.0
 tiktoken
 msgpack
 beautifulsoup4

infrastructure/terraform/lambda/_template/main.tf

@@ -33,6 +33,8 @@ module "lambda" {
 
   image_uri = local.image_uri
 
+  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
+  maximum_concurrency = var.maximum_concurrency
 
   environment = {
     STAGE = var.stage

infrastructure/terraform/lambda/_template/variables.tf

@@ -17,6 +17,11 @@ variable "image_digest" {
   description = "Image digest (sha256:...)"
 }
 
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
+}
 
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"

infrastructure/terraform/lambda/address2UPRN/main.tf

@@ -24,6 +24,9 @@ module "address2uprn" {
 
   timeout = 900
 
+  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
+  maximum_concurrency = var.maximum_concurrency
+
   environment = merge(
     {
       STAGE     = var.stage

infrastructure/terraform/lambda/address2UPRN/variables.tf

@@ -17,6 +17,11 @@ variable "image_digest" {
   description = "Image digest (sha256:...)"
 }
 
+variable "maximum_concurrency" {
+  type        = number
+  default     = 10 # null if you don't want to set it for this handler
+  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
+}
 
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"

infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf

@@ -44,5 +44,6 @@ module "sqs_trigger" {
   lambda_role_name = module.role.role_name
   queue_arn        = module.queue.queue_arn
 
-  batch_size = var.batch_size
+  batch_size           = var.batch_size
+  maximum_concurrency  = var.maximum_concurrency
 }

infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf

@@ -34,3 +34,9 @@ variable "batch_size" {
   type    = number
   default = 10
 }
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit."
+}

infrastructure/terraform/modules/lambda_sqs_trigger/main.tf

@@ -3,6 +3,13 @@ resource "aws_lambda_event_source_mapping" "this" {
   function_name    = var.lambda_arn
   batch_size       = var.batch_size
   enabled          = true
+
+  dynamic "scaling_config" {
+    for_each = var.maximum_concurrency != null ? [1] : []
+    content {
+      maximum_concurrency = var.maximum_concurrency
+    }
+  }
 }
 
 resource "aws_iam_role_policy" "allow_sqs" {

infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf

@@ -6,3 +6,9 @@ variable "batch_size" {
   type    = number
   default = 10
 }
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = null
+  description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit."
+}