From ad0c31fc3e869c8c5f08c4d368a698e56817e460 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 17 Apr 2026 14:19:12 +0000 Subject: [PATCH 1/4] IAM role allowing hubspot etl to send message to pashub queue --- .../terraform/lambda/hubspot_deal_etl/main.tf | 21 +++++++++++++++++++ .../terraform/lambda/pashub_to_ara/outputs.tf | 5 +++++ 2 files changed, 26 insertions(+) diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf index e8762337..32f0ba31 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf @@ -55,4 +55,25 @@ module "hubspot_deal_etl" { resource "aws_iam_role_policy_attachment" "lambda_s3_policy" { role = module.hubspot_deal_etl.role_name policy_arn = data.terraform_remote_state.shared.outputs.hubspot_etl_s3_read_and_write_arn +} + +# Create and attach S3 send policy for PasHub Fetcher queue +module "hubspot_deal_etl_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "hubspot-deal-etl-sqs-send-${var.stage}" + policy_description = "Allow Hubspot ETL Lambda to send messages to PasHub Fetcher queue" + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_arn + ] +} + +resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_sqs_send" { + role = module.lambda.role_name + policy_arn = module.hubspot_deal_etl_sqs_policy.policy_arn } \ No newline at end of file diff --git a/infrastructure/terraform/lambda/pashub_to_ara/outputs.tf b/infrastructure/terraform/lambda/pashub_to_ara/outputs.tf index d44b8763..584c9b63 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/outputs.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/outputs.tf @@ -2,3 +2,8 @@ output "pashub_to_ara_queue_url" { value = module.lambda.queue_url description = "URL of the PasHub to Ara SQS queue" } + +output "pashub_to_ara_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the PasHub to Ara SQS queue" +} From e61a50f0fd52134d9362fc796a0fdb90ae5ce159 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 17 Apr 2026 15:14:29 +0000 Subject: [PATCH 2/4] sqs policy terraform correction --- infrastructure/terraform/lambda/hubspot_deal_etl/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf index 32f0ba31..48dd6b78 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf @@ -74,6 +74,6 @@ module "hubspot_deal_etl_sqs_policy" { } resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_sqs_send" { - role = module.lambda.role_name + role = module.hubspot_deal_etl.role_name policy_arn = module.hubspot_deal_etl_sqs_policy.policy_arn } \ No newline at end of file From f6fa782026d9e68f4b86830834c6c983c4c85ae3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 17 Apr 2026 16:01:04 +0000 Subject: [PATCH 3/4] correct handler path in dockerfile --- backend/pashub_fetcher/handler/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/pashub_fetcher/handler/Dockerfile b/backend/pashub_fetcher/handler/Dockerfile index e450d340..d97d7a4a 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/backend/pashub_fetcher/handler/Dockerfile @@ -22,5 +22,4 @@ ENTRYPOINT ["python", "-m", "awslambdaric"] # ----------------------------- # Lambda handler # ----------------------------- -CMD ["backend.pashub_fetcher.handler.handler"] -# CMD ["backend.pashub_fetcher.handler.handler.handler"] \ No newline at end of file +CMD ["backend.pashub_fetcher.handler.handler.handler"] \ No newline at end of file From f5c038c98d237e5449dd8ad563b4fdb4641cc17a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 17 Apr 2026 16:48:05 +0000 Subject: [PATCH 4/4] fix lambda and docker issues --- backend/pashub_fetcher/handler/Dockerfile | 3 ++- backend/pashub_fetcher/handler/requirements.txt | 10 +++++++++- .../pashub_fetcher/local_handler/docker-compose.yml | 2 ++ .../local_handler/invoke_local_lambda.py | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/pashub_fetcher/handler/Dockerfile b/backend/pashub_fetcher/handler/Dockerfile index d97d7a4a..f8c2008c 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/backend/pashub_fetcher/handler/Dockerfile @@ -8,7 +8,8 @@ RUN chmod +x /usr/local/bin/aws-lambda-rie WORKDIR /var/task COPY utils/ utils/ -COPY backend/pashub_fetcher/ backend/pashub_fetcher/ +COPY backend/ backend/ +COPY datatypes/ datatypes/ COPY backend/pashub_fetcher/handler/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/backend/pashub_fetcher/handler/requirements.txt b/backend/pashub_fetcher/handler/requirements.txt index c4e416a8..ba235c7f 100644 --- a/backend/pashub_fetcher/handler/requirements.txt +++ b/backend/pashub_fetcher/handler/requirements.txt @@ -2,4 +2,12 @@ awslambdaric playwright==1.58.0 requests msal -openpyxl \ No newline at end of file +openpyxl +pydantic-settings +sqlalchemy +sqlmodel +psycopg2-binary +pytz +boto3==1.35.44 +pandas==2.2.2 +numpy<2.0 diff --git a/backend/pashub_fetcher/local_handler/docker-compose.yml b/backend/pashub_fetcher/local_handler/docker-compose.yml index 34ba9277..8b183d80 100644 --- a/backend/pashub_fetcher/local_handler/docker-compose.yml +++ b/backend/pashub_fetcher/local_handler/docker-compose.yml @@ -5,6 +5,8 @@ services: build: context: ../../../ dockerfile: backend/pashub_fetcher/handler/Dockerfile + entrypoint: ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] + command: ["backend.pashub_fetcher.handler.handler.handler"] ports: - "9000:8080" env_file: diff --git a/backend/pashub_fetcher/local_handler/invoke_local_lambda.py b/backend/pashub_fetcher/local_handler/invoke_local_lambda.py index 219446fd..5248a874 100644 --- a/backend/pashub_fetcher/local_handler/invoke_local_lambda.py +++ b/backend/pashub_fetcher/local_handler/invoke_local_lambda.py @@ -12,8 +12,9 @@ payload = { { "body": json.dumps( { - "pashub_link": "https://pashub.net/jobs/00000000-0000-0000-0000-000000000000/details", + "pashub_link": "https://google.co.uk", "uprn": "123456", + "hubspot_deal_id": "498926855369", } ) }