From 8b71fad8cc925bbbe5f3def953e6f1f3d2963f79 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 20 Apr 2026 09:24:36 +0000 Subject: [PATCH 01/10] correct environment variables --- .github/workflows/deploy_terraform.yml | 10 +--- .../terraform/lambda/pashub_to_ara/main.tf | 23 ++++++++ .../lambda/pashub_to_ara/variables.tf | 55 +++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 22f16fee..afa656d3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -332,17 +332,10 @@ jobs: ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }} dockerfile_path: backend/pashub_fetcher/handler/Dockerfile build_context: . - build_args: | - DEV_DB_HOST=$DEV_DB_HOST - DEV_DB_PORT=$DEV_DB_PORT - DEV_DB_NAME=$DEV_DB_NAME secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} - DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} - DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ @@ -362,6 +355,9 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} TF_VAR_sharepoint_client_id: ${{ secrets.SHAREPOINT_CLIENT_ID }} TF_VAR_sharepoint_client_secret: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} TF_VAR_sharepoint_tenant_id: ${{ secrets.SHAREPOINT_TENANT_ID }} diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/infrastructure/terraform/lambda/pashub_to_ara/main.tf index e898e949..0c652dc4 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/main.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/main.tf @@ -7,6 +7,14 @@ data "terraform_remote_state" "shared" { } } +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + module "lambda" { source = "../../modules/lambda_with_sqs" @@ -23,5 +31,20 @@ module "lambda" { environment = { STAGE = var.stage LOG_LEVEL = "info" + + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + + SHAREPOINT_CLIENT_ID = var.sharepoint_client_id + SHAREPOINT_CLIENT_SECRET = var.sharepoint_client_secret + DOMNA_SHAREPOINT_ID = var.domna_sharepoint_id + OSMOSIS_ACD_SHAREPOINT_ID = var.osmosis_acd_sharepoint_id + PRIVATE_PAY_SHAREPOINT_ID = var.private_pay_sharepoint_id + SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID = var.social_housing_wave_3_sharepoint_id + PASHUB_EMAIL = var.pashub_email + PASHUB_PASSWORD = var.pashub_password } } diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf index e7646811..f16b41ac 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf @@ -35,3 +35,58 @@ locals { output "resolved_image_uri" { value = local.image_uri } + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} + +variable "sharepoint_client_id" { + type = string + sensitive = true +} + +variable "sharepoint_client_secret" { + type = string + sensitive = true +} + +variable "domna_sharepoint_id" { + type = string + sensitive = true +} + +variable "osmosis_acd_sharepoint_id" { + type = string + sensitive = true +} + +variable "private_pay_sharepoint_id" { + type = string + sensitive = true +} + +variable "social_housing_wave_3_sharepoint_id" { + type = string + sensitive = true +} + +variable "pashub_email" { + type = string + sensitive = true +} + +variable "pashub_password" { + type = string + sensitive = true +} \ No newline at end of file From 28653208901db0f63604949c572953b945999ea0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 20 Apr 2026 11:08:33 +0000 Subject: [PATCH 02/10] default to not storing videos --- backend/pashub_fetcher/token_getter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/pashub_fetcher/token_getter.py b/backend/pashub_fetcher/token_getter.py index 5534d114..3f6c63e6 100644 --- a/backend/pashub_fetcher/token_getter.py +++ b/backend/pashub_fetcher/token_getter.py @@ -7,22 +7,23 @@ from utils.logger import setup_logger logger = setup_logger() -def get_token_from_local_storage(email: str, password: str) -> str: +def get_token_from_local_storage(email: str, password: str, record_video: bool = False) -> str: logger.info("Starting Playwright flow") - # For local testing / debugging, we save videos of the flow - video_dir = os.path.join(os.path.dirname(__file__), "videos") - os.makedirs(video_dir, exist_ok=True) - with sync_playwright() as p: browser = p.chromium.launch( headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"], ) + video_dir = None + if record_video: + video_dir = os.path.join(os.path.dirname(__file__), "videos") + os.makedirs(video_dir, exist_ok=True) + context = browser.new_context( record_video_dir=video_dir, - record_video_size={"width": 1280, "height": 720}, + record_video_size={"width": 1280, "height": 720} if record_video else None, ) page = context.new_page() @@ -71,8 +72,8 @@ def get_token_from_local_storage(email: str, password: str) -> str: raise Exception(f"Unexpected error: {str(e)}") finally: - logger.info("Closing browser context (saving video)...") context.close() browser.close() - logger.info(f"Video(s) saved in: {video_dir}") + if record_video and video_dir: + logger.info(f"Video(s) saved in: {video_dir}") From c1bd6851bf7a91e9852cc2580b7c75f60276a65a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 20 Apr 2026 11:10:05 +0000 Subject: [PATCH 03/10] correction --- backend/pashub_fetcher/token_getter.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/pashub_fetcher/token_getter.py b/backend/pashub_fetcher/token_getter.py index 3f6c63e6..ebc7506f 100644 --- a/backend/pashub_fetcher/token_getter.py +++ b/backend/pashub_fetcher/token_getter.py @@ -21,10 +21,13 @@ def get_token_from_local_storage(email: str, password: str, record_video: bool = video_dir = os.path.join(os.path.dirname(__file__), "videos") os.makedirs(video_dir, exist_ok=True) - context = browser.new_context( - record_video_dir=video_dir, - record_video_size={"width": 1280, "height": 720} if record_video else None, - ) + if record_video: + context = browser.new_context( + record_video_dir=video_dir, + record_video_size={"width": 1280, "height": 720}, + ) + else: + context = browser.new_context() page = context.new_page() try: From 42ea70e60fcf57053d492f8d63fbfae2f761328d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 20 Apr 2026 15:23:38 +0000 Subject: [PATCH 04/10] added additional deal properties --- backend/app/db/models/hubspot_deal_data.py | 4 ++++ etl/hubspot/hubspotClient.py | 8 ++++++-- etl/hubspot/hubspotDataTodB.py | 20 ++++++++++++++++---- etl/hubspot/hubspot_deal_differ.py | 8 ++++++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 758f688d..27649042 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -38,6 +38,10 @@ class HubspotDealData(SQLModel, table=True): dampmould_growth: Optional[str] = Field(default=None) damp_mould_and_repairs_comments: Optional[str] = Field(default=None) pre_sap: Optional[str] = Field(default=None) + batch: Optional[str] = Field(default=None) + block_reference: Optional[str] = Field(default=None) + epc_prn: Optional[str] = Field(default=None) + potential_post_sap_score_dropdown: Optional[str] = Field(default=None) coordinator: Optional[str] = Field(default=None) mtp_completion_date: Optional[datetime] = Field(default=None) mtp_re_model_completion_date: Optional[datetime] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 6bdf71ed..3a562700 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -254,12 +254,12 @@ class HubspotClient: "sharepoint_link", "dampmould_growth", "damp_mould_and_repairs_comments", - "pre_sap", + "pre_sap_score_dropdown", "coordinator", "mtp_completion_date", "mtp_re_model_completion_date", "ioe_v3_completion_date", - "proposed_measures", + "proposed_measures_dropdown", "approved_package", "designer", "design_completion_date", @@ -275,6 +275,10 @@ class HubspotClient: "confirmed_survey_time", "surveyed_date", "design_type", + "batch", + "block_reference", + "epc_prn", + "potential_post_sap_score_dropdown", ], ) ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 9756833b..eb8e2d14 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -159,7 +159,13 @@ class HubspotDataToDb: "damp_mould_and_repairs_comments": deal_data.get( "damp_mould_and_repairs_comments" ), - "pre_sap": deal_data.get("pre_sap"), + "pre_sap": deal_data.get("pre_sap_score_dropdown"), + "batch": deal_data.get("batch"), + "block_reference": deal_data.get("block_reference"), + "epc_prn": deal_data.get("epc_prn"), + "potential_post_sap_score_dropdown": deal_data.get( + "potential_post_sap_score_dropdown" + ), "coordinator": deal_data.get("coordinator"), "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), "mtp_re_model_completion_date": parse_hs_date( @@ -168,7 +174,7 @@ class HubspotDataToDb: "ioe_v3_completion_date": parse_hs_date( deal_data.get("ioe_v3_completion_date") ), - "proposed_measures": deal_data.get("proposed_measures"), + "proposed_measures": deal_data.get("proposed_measures_dropdown"), "approved_package": deal_data.get("approved_package"), "designer": deal_data.get("designer"), "design_completion_date": parse_hs_date( @@ -228,7 +234,13 @@ class HubspotDataToDb: damp_mould_and_repairs_comments=deal_data.get( "damp_mould_and_repairs_comments" ), - pre_sap=deal_data.get("pre_sap"), + pre_sap=deal_data.get("pre_sap_score_dropdown"), + batch=deal_data.get("batch"), + block_reference=deal_data.get("block_reference"), + epc_prn=deal_data.get("epc_prn"), + potential_post_sap_score_dropdown=deal_data.get( + "potential_post_sap_score_dropdown" + ), coordinator=deal_data.get("coordinator"), mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), mtp_re_model_completion_date=parse_hs_date( @@ -237,7 +249,7 @@ class HubspotDataToDb: ioe_v3_completion_date=parse_hs_date( deal_data.get("ioe_v3_completion_date") ), - proposed_measures=deal_data.get("proposed_measures"), + proposed_measures=deal_data.get("proposed_measures_dropdown"), approved_package=deal_data.get("approved_package"), designer=deal_data.get("designer"), design_completion_date=parse_hs_date( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 74c8264d..cf9ad1ee 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -62,9 +62,13 @@ class HubspotDealDiffer: "sharepoint_link": "sharepoint_link", "dampmould_growth": "dampmould_growth", "damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments", - "pre_sap": "pre_sap", + "pre_sap_score_dropdown": "pre_sap", + "batch": "batch", + "block_reference": "block_reference", + "epc_prn": "epc_prn", + "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", "coordinator": "coordinator", - "proposed_measures": "proposed_measures", + "proposed_measures_dropdown": "proposed_measures", "approved_package": "approved_package", "designer": "designer", "actual_measures_installed": "actual_measures_installed", From 902e5cd8dfb0696b80190eb66d7a96f44f8f46fc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 20 Apr 2026 15:44:54 +0000 Subject: [PATCH 05/10] added more deal properties --- backend/app/db/models/hubspot_deal_data.py | 4 ++++ etl/hubspot/hubspotClient.py | 4 ++++ etl/hubspot/hubspotDataTodB.py | 8 ++++++++ etl/hubspot/hubspot_deal_differ.py | 4 ++++ etl/hubspot/scripts/scraper/bulk_load.py | 4 ++-- 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 27649042..fa508fbe 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -42,6 +42,10 @@ class HubspotDealData(SQLModel, table=True): block_reference: Optional[str] = Field(default=None) epc_prn: Optional[str] = Field(default=None) potential_post_sap_score_dropdown: Optional[str] = Field(default=None) + ei_score: Optional[str] = Field(default=None) + ei_score__potential_: Optional[str] = Field(default=None) + epc_sap_score: Optional[str] = Field(default=None) + epc_sap_score__potential_: Optional[str] = Field(default=None) coordinator: Optional[str] = Field(default=None) mtp_completion_date: Optional[datetime] = Field(default=None) mtp_re_model_completion_date: Optional[datetime] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 3a562700..df28e4d6 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -279,6 +279,10 @@ class HubspotClient: "block_reference", "epc_prn", "potential_post_sap_score_dropdown", + "ei_score", + "ei_score__potential_", + "epc_sap_score", + "epc_sap_score__potential_", ], ) ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index eb8e2d14..a2eb24c2 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -166,6 +166,10 @@ class HubspotDataToDb: "potential_post_sap_score_dropdown": deal_data.get( "potential_post_sap_score_dropdown" ), + "ei_score": deal_data.get("ei_score"), + "ei_score__potential_": deal_data.get("ei_score__potential_"), + "epc_sap_score": deal_data.get("epc_sap_score"), + "epc_sap_score__potential_": deal_data.get("epc_sap_score__potential_"), "coordinator": deal_data.get("coordinator"), "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), "mtp_re_model_completion_date": parse_hs_date( @@ -241,6 +245,10 @@ class HubspotDataToDb: potential_post_sap_score_dropdown=deal_data.get( "potential_post_sap_score_dropdown" ), + ei_score=deal_data.get("ei_score"), + ei_score__potential_=deal_data.get("ei_score__potential_"), + epc_sap_score=deal_data.get("epc_sap_score"), + epc_sap_score__potential_=deal_data.get("epc_sap_score__potential_"), coordinator=deal_data.get("coordinator"), mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), mtp_re_model_completion_date=parse_hs_date( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index cf9ad1ee..80c1fe04 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -67,6 +67,10 @@ class HubspotDealDiffer: "block_reference": "block_reference", "epc_prn": "epc_prn", "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", + "ei_score": "ei_score", + "ei_score__potential_": "ei_score__potential_", + "epc_sap_score": "epc_sap_score", + "epc_sap_score__potential_": "epc_sap_score__potential_", "coordinator": "coordinator", "proposed_measures_dropdown": "proposed_measures", "approved_package": "approved_package", diff --git a/etl/hubspot/scripts/scraper/bulk_load.py b/etl/hubspot/scripts/scraper/bulk_load.py index f0529905..91aa89e2 100644 --- a/etl/hubspot/scripts/scraper/bulk_load.py +++ b/etl/hubspot/scripts/scraper/bulk_load.py @@ -9,8 +9,8 @@ PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value companies = list( [ # Companies.THE_GUINESS_PARTNERSHIP, - Companies.SOUTHERN_HOUSING_GROUP, - # Companies.CALICO_HOMES, + # Companies.SOUTHERN_HOUSING_GROUP, + Companies.CALICO_HOMES, ] ) From 30ac251705ddb1dc5337a388743d2dd2f9c3a49c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 21 Apr 2026 08:35:52 +0000 Subject: [PATCH 06/10] limit pashub lambda concurrency to 1 --- infrastructure/terraform/lambda/pashub_to_ara/main.tf | 2 ++ infrastructure/terraform/lambda/pashub_to_ara/variables.tf | 6 ++++++ infrastructure/terraform/modules/lambda_service/main.tf | 2 ++ .../terraform/modules/lambda_service/variables.tf | 6 ++++++ infrastructure/terraform/modules/lambda_with_sqs/main.tf | 3 ++- .../terraform/modules/lambda_with_sqs/variables.tf | 6 ++++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/infrastructure/terraform/lambda/pashub_to_ara/main.tf index 0c652dc4..ae719a99 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/main.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/main.tf @@ -26,6 +26,8 @@ module "lambda" { # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size environment = { diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf index f16b41ac..e68a26b6 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf @@ -23,6 +23,12 @@ variable "maximum_concurrency" { description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } +variable "reserved_concurrent_executions" { + type = number + default = 1 + description = "Reserved concurrency. Defaults to 1 to prevent concurrent Playwright browser collisions." +} + variable "batch_size" { type = number default = 1 diff --git a/infrastructure/terraform/modules/lambda_service/main.tf b/infrastructure/terraform/modules/lambda_service/main.tf index 8a159db1..3250110b 100644 --- a/infrastructure/terraform/modules/lambda_service/main.tf +++ b/infrastructure/terraform/modules/lambda_service/main.tf @@ -9,6 +9,8 @@ resource "aws_lambda_function" "this" { memory_size = var.memory_size publish = true + reserved_concurrent_executions = var.reserved_concurrent_executions + environment { variables = var.environment } diff --git a/infrastructure/terraform/modules/lambda_service/variables.tf b/infrastructure/terraform/modules/lambda_service/variables.tf index 43def6ad..46241f30 100644 --- a/infrastructure/terraform/modules/lambda_service/variables.tf +++ b/infrastructure/terraform/modules/lambda_service/variables.tf @@ -16,3 +16,9 @@ variable "environment" { type = map(string) default = {} } + +variable "reserved_concurrent_executions" { + type = number + default = -1 + description = "Reserved concurrency for the Lambda function. -1 = unreserved (default). 0 = throttle all. 1+ = hard limit." +} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/modules/lambda_with_sqs/main.tf index 35626487..97f86793 100644 --- a/infrastructure/terraform/modules/lambda_with_sqs/main.tf +++ b/infrastructure/terraform/modules/lambda_with_sqs/main.tf @@ -31,7 +31,8 @@ module "lambda" { timeout = var.timeout memory_size = var.memory_size - environment = var.environment + environment = var.environment + reserved_concurrent_executions = var.reserved_concurrent_executions } ############################################ diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf index 7c2832d2..90585e92 100644 --- a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf @@ -40,3 +40,9 @@ variable "maximum_concurrency" { default = null description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit." } + +variable "reserved_concurrent_executions" { + type = number + default = -1 + description = "Reserved concurrency for the Lambda function. -1 = unreserved. 1 = single-threaded." +} From 5966757051699b7f6f5e07a9b6652a34bb060aa9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 21 Apr 2026 08:58:51 +0000 Subject: [PATCH 07/10] add reserved_concurrent_executions to template lambda terraform --- infrastructure/terraform/lambda/_template/main.tf | 2 ++ infrastructure/terraform/lambda/_template/variables.tf | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 81b1c7f1..1cd63ffe 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -36,6 +36,8 @@ module "lambda" { # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size environment = { diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 0a3092ee..daaa0b7c 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -23,6 +23,12 @@ variable "maximum_concurrency" { description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } +variable "reserved_concurrent_executions" { + type = number + default = -1 + description = "Reserved concurrency for the Lambda function. -1 = unreserved. 1+ = hard limit across all triggers." +} + variable "batch_size" { type = number default = 1 From 9b319ba7995929ba94832521e23a2939289b1367 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 21 Apr 2026 13:00:33 +0000 Subject: [PATCH 08/10] doesn't need this library --- backend/app/bulk_uploads/router.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/bulk_uploads/router.py b/backend/app/bulk_uploads/router.py index ca1e7b79..8f0c751f 100644 --- a/backend/app/bulk_uploads/router.py +++ b/backend/app/bulk_uploads/router.py @@ -8,7 +8,6 @@ from backend.app.config import get_settings from backend.app.db.connection import get_db_session from backend.app.db.models.bulk_address_uploads import BulkAddressUpload from backend.app.bulk_uploads.schema import PostcodeSplitterTriggerRequest -from utils.s3 import parse_s3_uri, read_csv_from_s3 router = APIRouter( From 3814eed926d533f3fe6bea4db04b12b8e8706d57 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 21 Apr 2026 14:05:10 +0000 Subject: [PATCH 09/10] correct hubspot deal differ logic --- etl/hubspot/hubspot_deal_differ.py | 4 ++-- etl/hubspot/tests/test_hubspot_deal_differ.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 80c1fe04..fa5bbe42 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -166,7 +166,7 @@ class HubspotDealDiffer: @staticmethod def _design_completed(new_deal: Dict[str, str], old_deal: HubspotDealData) -> bool: - new_status: str = new_deal.get("coordination_status") or "" + new_status: str = new_deal.get("design_status") or "" return ( new_status != "" and new_status.lower() == HubspotDealDiffer.RETROFIT_DESIGN_COMPLETE @@ -177,7 +177,7 @@ class HubspotDealDiffer: def _lodgement_completed( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - new_status: str = new_deal.get("coordination_status") or "" + new_status: str = new_deal.get("lodgement_status") or "" return ( new_status != "" and new_status.lower() in HubspotDealDiffer.LODGEMENT_COMPLETE diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 69f7668b..0523c982 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -90,8 +90,8 @@ def test_pashub_trigger__pashub_link_changed__returns_true( @pytest.mark.parametrize( "coordination_status,expected", [ - ("v1 ioe/mtp complete", True), - ("v2 ioe/mtp complete", True), + ("(v1) ioe/mtp complete", True), + ("(v2) ioe/mtp complete", True), ], ) def test_pashub_trigger__coordination_completed_and_pashub_link_set__returns_true( From 7ddd95f28fb738a1548723fd7d3399fbee348be8 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 21 Apr 2026 16:17:30 +0000 Subject: [PATCH 10/10] debug playwright browser launcher --- backend/pashub_fetcher/token_getter.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/pashub_fetcher/token_getter.py b/backend/pashub_fetcher/token_getter.py index ebc7506f..2e2d1440 100644 --- a/backend/pashub_fetcher/token_getter.py +++ b/backend/pashub_fetcher/token_getter.py @@ -7,14 +7,25 @@ from utils.logger import setup_logger logger = setup_logger() -def get_token_from_local_storage(email: str, password: str, record_video: bool = False) -> str: +def get_token_from_local_storage( + email: str, password: str, record_video: bool = False +) -> str: logger.info("Starting Playwright flow") with sync_playwright() as p: + logger.info("Playwright server started") + browser = p.chromium.launch( headless=True, - args=["--no-sandbox", "--disable-dev-shm-usage"], + args=[ + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--single-process", + "--no-zygote", + ], ) + logger.info("Chromium launched successfully") video_dir = None if record_video: @@ -29,6 +40,7 @@ def get_token_from_local_storage(email: str, password: str, record_video: bool = else: context = browser.new_context() page = context.new_page() + logger.info("Page created") try: logger.info("Navigating to site...")