From 23d7b22b548aef03820723eb118f812c70c44b14 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 2 Apr 2026 17:30:38 +0000 Subject: [PATCH 1/9] save for easter weekend --- backend/address2UPRN/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/address2UPRN/README.md b/backend/address2UPRN/README.md index 646fec01..b17b36d1 100644 --- a/backend/address2UPRN/README.md +++ b/backend/address2UPRN/README.md @@ -32,9 +32,9 @@ Step 3) Alright, now lets make the input for postcode-splitter sqs to get the ba postcode-splitter-sqs => https://eu-west-2.console.aws.amazon.com/sqs/v3/home?region=eu-west-2#/queues/https%3A%2F%2Fsqs.eu-west-2.amazonaws.com%2F337213553626%2Fpostcode-splitter-queue-dev { - "task_id": "ea615ac3-ac28-46c4-8bff-2431c5b9c13d", - "sub_task_id": "85a23b67-8f18-4299-9bf0-69bfb87adbc7", - "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/eon/eon(Sheet1).csv" + "sub_task_id": "c5afbd49-f0cd-4930-82bf-bafc5243a34a", + "task_id": "67a4b3f0-cc7a-4e8a-b314-deb783e0eedb", + "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/eon/pickering ferens/Pickering Ferens - SHDF W3 Post Bid Stage MDS - Template - Vr4(in).csv" } Each batch of csv should be saved in retrofit-data-dev/ara_postcode_splitter_batches///.csv From 959867f5ee887803b5912355db2a5cc3acf7ebff Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 09:51:26 +0000 Subject: [PATCH 2/9] maximum concurrent --- infrastructure/terraform/lambda/_template/variables.tf | 2 +- infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf | 4 ++-- infrastructure/terraform/lambda/ordnanceSurvey/variables.tf | 2 +- infrastructure/terraform/lambda/pashub_to_ara/variables.tf | 2 +- .../terraform/modules/lambda_sqs_trigger/variables.tf | 2 +- infrastructure/terraform/modules/lambda_with_sqs/variables.tf | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index ae588840..c4408e0f 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf index 285b6a4c..27c9424b 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf @@ -19,13 +19,13 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } variable "batch_size" { type = number - default = 1 + default = 10 } locals { diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index 936aebc9..e2cfd65e 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf index e7646811..fe92d645 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf index c3127c74..0b8dffc0 100644 --- a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf +++ b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf @@ -9,6 +9,6 @@ variable "batch_size" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit." } diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf index 7c2832d2..6d992f3b 100644 --- a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf @@ -37,6 +37,6 @@ variable "batch_size" { variable "maximum_concurrency" { type = number - default = null + default = 1 description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit." } From 8d6be23084a73e05929c8ccd2f1bb3263f3be6f7 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 09:56:38 +0000 Subject: [PATCH 3/9] maximum concurrency --- infrastructure/terraform/lambda/ordnanceSurvey/variables.tf | 2 +- infrastructure/terraform/lambda/pashub_to_ara/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf index e2cfd65e..936aebc9 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = 1 + default = null description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf index fe92d645..e7646811 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = 1 + default = null description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } From 2c39f64731382497b3d53fb5e470c4bfb32dc8ae Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 09:58:22 +0000 Subject: [PATCH 4/9] maximum concurrency --- infrastructure/terraform/modules/lambda_with_sqs/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf index 6d992f3b..7c2832d2 100644 --- a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf @@ -37,6 +37,6 @@ variable "batch_size" { variable "maximum_concurrency" { type = number - default = 1 + default = null description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit." } From 469401aabd341089a8a9813decc86f213e2a1a78 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 09:59:41 +0000 Subject: [PATCH 5/9] maximum concurrency --- .../terraform/modules/lambda_sqs_trigger/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf index 0b8dffc0..c3127c74 100644 --- a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf +++ b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf @@ -9,6 +9,6 @@ variable "batch_size" { variable "maximum_concurrency" { type = number - default = 1 + default = null description = "Maximum number of concurrent Lambda invocations from SQS. null = no limit." } From acbd0cfc35e3d382b73802e6d5fb6b44a1d73951 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 10:53:57 +0000 Subject: [PATCH 6/9] maximum concurrency to 2 --- infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf index 27c9424b..499f9cc5 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf @@ -19,13 +19,13 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = 1 + default = 2 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } variable "batch_size" { type = number - default = 10 + default = 1 } locals { From 426d907ce766059646dce102c710d1fe83d790c0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 10:54:17 +0000 Subject: [PATCH 7/9] 2 lambda b ut 1 batch size --- infrastructure/terraform/lambda/_template/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index c4408e0f..0a3092ee 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -19,7 +19,7 @@ variable "image_digest" { variable "maximum_concurrency" { type = number - default = 1 + default = 2 description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } From 25e08f3e964b84e79e9e9b42964ec57d89423cf2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 11:08:19 +0000 Subject: [PATCH 8/9] 2 lambda b ut 1 batch size --- infrastructure/terraform/lambda/_template/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 0a3092ee..63f60cb6 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -25,7 +25,7 @@ variable "maximum_concurrency" { variable "batch_size" { type = number - default = 1 + default = 5 } locals { From 34414d14a50df31b487778efce885e613bb14039 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 7 Apr 2026 11:10:15 +0000 Subject: [PATCH 9/9] change to sensiable concurrecy and batch size --- infrastructure/terraform/lambda/_template/variables.tf | 2 +- infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 63f60cb6..0a3092ee 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -25,7 +25,7 @@ variable "maximum_concurrency" { variable "batch_size" { type = number - default = 5 + default = 1 } locals { diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf index 499f9cc5..84f0e567 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf @@ -25,7 +25,7 @@ variable "maximum_concurrency" { variable "batch_size" { type = number - default = 1 + default = 5 } locals {