From 676022a4c054d7301b1d7542b582fadbb63705b4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 12:53:37 +0000 Subject: [PATCH 01/28] =?UTF-8?q?Fix=20coordination/design=20field=20names?= =?UTF-8?q?=20and=20add=20MagicPlan=20trigger=20to=20HubspotDealDiffer=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 6 ++ etl/hubspot/tests/test_hubspot_deal_differ.py | 76 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 9e7069fc..a53df4f7 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -194,6 +194,12 @@ class HubspotDealDiffer: and new_status != old_deal.design_status ) + @staticmethod + def check_for_magicplan_trigger( + new_deal: Dict[str, str], old_deal: HubspotDealData + ) -> bool: + raise NotImplementedError + @staticmethod def _lodgement_completed( new_deal: Dict[str, str], old_deal: HubspotDealData diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 0523c982..273a82a0 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -109,7 +109,7 @@ def test_pashub_trigger__coordination_completed_and_pashub_link_set__returns_tru new_deal = make_new_deal( deal_id, pashub_link="www.google.co.uk", - coordination_status=coordination_status, + **{"coordination_status__stage_1_": coordination_status}, ) assert ( @@ -156,7 +156,7 @@ def test_pashub_trigger__design_completed_and_pashub_link_set__returns_true() -> new_deal = make_new_deal( deal_id, pashub_link="www.google.co.uk", - design_status="uploaded", + retrofit_design_status="uploaded", ) assert ( @@ -177,7 +177,7 @@ def test_pashub_trigger__design_completed_and_pashub_link_not_set__returns_false new_deal = make_new_deal( deal_id, - design_status="uploaded", + retrofit_design_status="uploaded", ) assert ( @@ -270,6 +270,76 @@ def test_pashub_trigger__coordination_design_lodgement_not_completed_and_pashub_ ) +# ========================== +# MAGICPLAN TRIGGER TESTS +# ========================== + + +def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, coordination_status="in progress") + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is True + + +def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__returns_false() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal( + id=deal_id, + coordination_status="(v1) ioe/mtp complete", + outcome="pending", + ) + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, + outcome="won", + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is False + + +def test_magicplan_trigger__transitions_to_non_complete_coordination_status__returns_false() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, coordination_status="in progress") + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "design submitted"}, + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is False + + # ======================= # DB UPDATE TRIGGER TESTS # ======================= From 69faa530a4c5a4dce34a00919baf965e2f59943a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 12:55:23 +0000 Subject: [PATCH 02/28] =?UTF-8?q?Fix=20coordination/design=20field=20names?= =?UTF-8?q?=20and=20add=20MagicPlan=20trigger=20to=20HubspotDealDiffer=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index a53df4f7..5435a46d 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -178,7 +178,7 @@ class HubspotDealDiffer: def _coordination_completed( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - new_status: str = new_deal.get("coordination_status") or "" + new_status: str = new_deal.get("coordination_status__stage_1_") or "" return ( new_status != "" and new_status.lower() in HubspotDealDiffer.COORDINATION_COMPLETE @@ -187,7 +187,7 @@ class HubspotDealDiffer: @staticmethod def _design_completed(new_deal: Dict[str, str], old_deal: HubspotDealData) -> bool: - new_status: str = new_deal.get("design_status") or "" + new_status: str = new_deal.get("retrofit_design_status") or "" return ( new_status != "" and new_status.lower() == HubspotDealDiffer.RETROFIT_DESIGN_COMPLETE @@ -198,7 +198,12 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - raise NotImplementedError + new_status = (new_deal.get("coordination_status__stage_1_") or "").lower() + old_status = (old_deal.coordination_status or "").lower() + return ( + new_status in HubspotDealDiffer.COORDINATION_COMPLETE + and old_status not in HubspotDealDiffer.COORDINATION_COMPLETE + ) @staticmethod def _lodgement_completed( From 489b0ba30eb730139916901da7b585627428c3e9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:05:38 +0000 Subject: [PATCH 03/28] =?UTF-8?q?Add=20MagicPlan=20SQS=20trigger=20to=20Hu?= =?UTF-8?q?bSpot=20orchestrator=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/config.py | 1 + .../hubspot_trigger_orchestrator/__init__.py | 0 .../tests/__init__.py | 0 .../tests/test_orchestrator.py | 148 ++++++++++++++++++ etl/hubspot/scripts/scraper/main.py | 21 +++ 5 files changed, 170 insertions(+) create mode 100644 backend/hubspot_trigger_orchestrator/__init__.py create mode 100644 backend/hubspot_trigger_orchestrator/tests/__init__.py create mode 100644 backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py diff --git a/backend/app/config.py b/backend/app/config.py index e939d6e4..21f12902 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -39,6 +39,7 @@ class Settings(BaseSettings): ENGINE_SQS_URL: str = "changeme" CATEGORISATION_SQS_URL: str = "changeme" PASHUB_TO_ARA_SQS_URL: str = "changeme" + MAGICPLAN_SQS_URL: str = "changeme" POSTCODE_SPLITTER_SQS_URL: str = "changeme" COMBINER_SQS_URL: str = "changeme" diff --git a/backend/hubspot_trigger_orchestrator/__init__.py b/backend/hubspot_trigger_orchestrator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/hubspot_trigger_orchestrator/tests/__init__.py b/backend/hubspot_trigger_orchestrator/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py b/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py new file mode 100644 index 00000000..6d18c4b4 --- /dev/null +++ b/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py @@ -0,0 +1,148 @@ +import json +import uuid +from typing import Any, Dict, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from etl.hubspot.scripts.scraper.main import handler + +COORDINATION_COMPLETE = "(v1) ioe/mtp complete" +DEAL_NAME = "123 Main Street" +UPRN = "12345678" +DEAL_ID = "999" +MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" + + +def make_hubspot_deal( + coordination_status: Optional[str] = None, **kwargs: Any +) -> Dict[str, Any]: + deal: Dict[str, Any] = { + "hs_object_id": DEAL_ID, + "dealname": DEAL_NAME, + "pashub_link": None, + **kwargs, + } + if coordination_status is not None: + deal["coordination_status__stage_1_"] = coordination_status + return deal + + +def make_db_deal(coordination_status: Optional[str] = None, **kwargs: Any) -> HubspotDealData: + return HubspotDealData( + id=uuid.uuid4(), + deal_id=DEAL_ID, + coordination_status=coordination_status, + **kwargs, + ) + + +def run_handler( + hubspot_deal: Dict[str, Any], + db_deal: Optional[HubspotDealData], + listing: Optional[dict], +) -> MagicMock: + mock_sqs = MagicMock() + mock_sqs.send_message.return_value = {"MessageId": "test-id"} + + with ( + patch("etl.hubspot.scripts.scraper.main.HubspotDataToDb") as mock_db_cls, + patch("etl.hubspot.scripts.scraper.main.HubspotClient") as mock_hs_cls, + patch("etl.hubspot.scripts.scraper.main.boto3") as mock_boto3, + patch("etl.hubspot.scripts.scraper.main.get_settings") as mock_settings, + ): + mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal + mock_db_cls.return_value.upsert_deal.return_value = None + mock_hs_cls.return_value.get_deal_and_company_and_listing.return_value = ( + hubspot_deal, + None, + listing, + ) + mock_boto3.client.return_value = mock_sqs + mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL + mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = "https://sqs.test/pashub" + + handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") + + return mock_sqs + + +# ======================= +# NEW DEAL PATH +# ======================= + + +def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + listing = {"national_uprn": UPRN} + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + ) + + +def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status="in progress") + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() + + +def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": None}), + ) + + +# ======================= +# EXISTING DEAL PATH +# ======================= + + +def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() -> None: + # Arrange + db_deal = make_db_deal(coordination_status="in progress") + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + listing = {"national_uprn": UPRN} + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + ) + + +def test_existing_deal_already_in_coordination_complete_unrelated_change__no_sqs_message() -> None: + # Arrange + db_deal = make_db_deal(coordination_status=COORDINATION_COMPLETE, dealname="Old Name") + hubspot_deal = make_hubspot_deal( + coordination_status=COORDINATION_COMPLETE, dealname="New Name" + ) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 3ed208a2..cd76e26f 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -56,6 +56,13 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) _trigger_pashub_fetcher(sqs_client, hubspot_deal_id, hubspot_deal) + + coordination_status = (hubspot_deal.get("coordination_status__stage_1_") or "").lower() + if coordination_status in HubspotDealDiffer.COORDINATION_COMPLETE: + logger.info( + f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) else: # Deal already in db, check whether anything has changed logger.info( @@ -97,9 +104,23 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) + if HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=hubspot_deal, old_deal=db_deal + ): + logger.info( + f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + print("done") +def _trigger_magicplan_fetcher( + sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] +) -> None: + raise NotImplementedError + + def _trigger_pashub_fetcher( sqs_client: Any, deal_id: str, hubspot_deal: Dict[str, str] ) -> None: From a1a445f6f270f62b148e7d3f79eace348213e893 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:06:43 +0000 Subject: [PATCH 04/28] =?UTF-8?q?Add=20MagicPlan=20SQS=20trigger=20to=20Hu?= =?UTF-8?q?bSpot=20orchestrator=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/scripts/scraper/main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index cd76e26f..a39e8b37 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -118,7 +118,17 @@ def handler(body: dict[str, Any], context: Any) -> None: def _trigger_magicplan_fetcher( sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] ) -> None: - raise NotImplementedError + message_body = { + "address": hubspot_deal.get("dealname"), + "uprn": listing.get("national_uprn") if listing else None, + } + response = sqs_client.send_message( + QueueUrl=get_settings().MAGICPLAN_SQS_URL, + MessageBody=json.dumps(message_body), + ) + logger.info( + f"Sent message to MagicPlan queue. MessageId: {response['MessageId']}" + ) def _trigger_pashub_fetcher( From ed68a10127b132d883b927655c5f0ce2cdc41815 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:07:36 +0000 Subject: [PATCH 05/28] magic plan client terraform --- .../terraform/lambda/magic_plan/main.tf | 41 +++++++++++++++++++ .../terraform/lambda/magic_plan/outputs.tf | 9 ++++ .../terraform/lambda/magic_plan/provider.tf | 16 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 infrastructure/terraform/lambda/magic_plan/main.tf create mode 100644 infrastructure/terraform/lambda/magic_plan/outputs.tf create mode 100644 infrastructure/terraform/lambda/magic_plan/provider.tf diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf new file mode 100644 index 00000000..56adac1b --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -0,0 +1,41 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "magic_plan" + stage = var.stage + + image_uri = local.image_uri + + maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + MAGICPLAN_CUSTOMER_ID = var.magicplan_customer_id + MAGICPLAN_API_KEY = var.magicplan_api_key + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + } +} diff --git a/infrastructure/terraform/lambda/magic_plan/outputs.tf b/infrastructure/terraform/lambda/magic_plan/outputs.tf new file mode 100644 index 00000000..2082933f --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/outputs.tf @@ -0,0 +1,9 @@ +output "magic_plan_queue_url" { + value = module.lambda.queue_url + description = "URL of the MagicPlan SQS queue" +} + +output "magic_plan_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the MagicPlan SQS queue" +} diff --git a/infrastructure/terraform/lambda/magic_plan/provider.tf b/infrastructure/terraform/lambda/magic_plan/provider.tf new file mode 100644 index 00000000..9e7020ac --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "magic-plan-hubspot-trigger-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} From fd77fa51fdad5f1ea5dd3a6859f3051dd0665d84 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:07:53 +0000 Subject: [PATCH 06/28] magic plan client terraform --- .../terraform/lambda/magic_plan/variables.tf | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 infrastructure/terraform/lambda/magic_plan/variables.tf diff --git a/infrastructure/terraform/lambda/magic_plan/variables.tf b/infrastructure/terraform/lambda/magic_plan/variables.tf new file mode 100644 index 00000000..03f88e75 --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/variables.tf @@ -0,0 +1,68 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null + description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} + +variable "magicplan_customer_id" { + type = string + sensitive = true +} + +variable "magicplan_api_key" { + type = string + sensitive = true +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} From feaa1ea68093f74087952597be691ee5a387fc8a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:12:13 +0000 Subject: [PATCH 07/28] Add MagicPlan Lambda Dockerfile, CI/CD jobs, and SQS IAM wiring in hubspot_deal_etl --- .github/workflows/deploy_terraform.yml | 40 ++++++++++++++++++- backend/magic_plan/handler/Dockerfile | 26 ++++++++++++ backend/magic_plan/handler/requirements.txt | 7 ++++ .../terraform/lambda/hubspot_deal_etl/main.tf | 26 +++++++++++- 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 backend/magic_plan/handler/Dockerfile create mode 100644 backend/magic_plan/handler/requirements.txt diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 398232c6..e0343974 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -537,11 +537,49 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # Build MagicPlan Lambda image + # ============================================================ + magic_plan_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/magic_plan/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy MagicPlan Lambda + # ============================================================ + magic_plan_lambda: + needs: [magic_plan_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: magic_plan + lambda_path: infrastructure/terraform/lambda/magic_plan + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.magic_plan_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }} + TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }} + # ============================================================ # Deploy Hubspot ETL Lambda # ============================================================ hubspot_etl_lambda: - needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda] + needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda, magic_plan_lambda] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: hubspot-etl-to-ara diff --git a/backend/magic_plan/handler/Dockerfile b/backend/magic_plan/handler/Dockerfile new file mode 100644 index 00000000..7c83ebe6 --- /dev/null +++ b/backend/magic_plan/handler/Dockerfile @@ -0,0 +1,26 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy + +# Install AWS Lambda RIE +ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie +RUN chmod +x /usr/local/bin/aws-lambda-rie + +# Set working directory (Lambda task root) +WORKDIR /var/task + +COPY backend/magic_plan/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utils/ utils/ +COPY backend/ backend/ +COPY datatypes/ datatypes/ + +# Local lambda entrypoint +# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] + +# AWS lambda entrypoint +ENTRYPOINT ["python", "-m", "awslambdaric"] + +# ----------------------------- +# Lambda handler +# ----------------------------- +CMD ["backend.magic_plan.handler.handler"] diff --git a/backend/magic_plan/handler/requirements.txt b/backend/magic_plan/handler/requirements.txt new file mode 100644 index 00000000..cfacf455 --- /dev/null +++ b/backend/magic_plan/handler/requirements.txt @@ -0,0 +1,7 @@ +awslambdaric +requests +sqlalchemy==2.0.36 +sqlmodel +psycopg2-binary==2.9.10 +pydantic-settings==2.6.0 +boto3==1.35.44 diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf index 48dd6b78..800dc3b6 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf @@ -12,7 +12,16 @@ data "terraform_remote_state" "pashub_to_ara" { config = { bucket = "pashub-to-ara-terraform-state" key = "env:/${var.stage}/terraform.tfstate" - region = "eu-west-2" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "magic_plan" { + backend = "s3" + config = { + bucket = "magic-plan-hubspot-trigger-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" } } @@ -49,6 +58,7 @@ module "hubspot_deal_etl" { HUBSPOT_API_KEY = var.hubspot_api_key PASHUB_TO_ARA_SQS_URL = data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_url + MAGICPLAN_SQS_URL = data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_url } } @@ -76,4 +86,18 @@ module "hubspot_deal_etl_sqs_policy" { resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_sqs_send" { role = module.hubspot_deal_etl.role_name policy_arn = module.hubspot_deal_etl_sqs_policy.policy_arn +} + +module "hubspot_deal_etl_magicplan_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "hubspot-deal-etl-magicplan-sqs-send-${var.stage}" + policy_description = "Allow HubSpot ETL Lambda to send messages to MagicPlan queue" + actions = ["sqs:SendMessage"] + resources = [data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_arn] +} + +resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_magicplan_sqs_send" { + role = module.hubspot_deal_etl.role_name + policy_arn = module.hubspot_deal_etl_magicplan_sqs_policy.policy_arn } \ No newline at end of file From e30e06cb6e61f24668bba039e0fb17c36f4a9307 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:24:04 +0000 Subject: [PATCH 08/28] simplify dockerfile as playwright not used --- backend/magic_plan/handler/Dockerfile | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/backend/magic_plan/handler/Dockerfile b/backend/magic_plan/handler/Dockerfile index 7c83ebe6..ffd85c02 100644 --- a/backend/magic_plan/handler/Dockerfile +++ b/backend/magic_plan/handler/Dockerfile @@ -1,10 +1,5 @@ -FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy +FROM public.ecr.aws/lambda/python:3.11 -# Install AWS Lambda RIE -ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie -RUN chmod +x /usr/local/bin/aws-lambda-rie - -# Set working directory (Lambda task root) WORKDIR /var/task COPY backend/magic_plan/handler/requirements.txt . @@ -14,13 +9,4 @@ COPY utils/ utils/ COPY backend/ backend/ COPY datatypes/ datatypes/ -# Local lambda entrypoint -# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] - -# AWS lambda entrypoint -ENTRYPOINT ["python", "-m", "awslambdaric"] - -# ----------------------------- -# Lambda handler -# ----------------------------- CMD ["backend.magic_plan.handler.handler"] From 74b3a7f297b90535f32208189c6d9a87c24f806a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:42:57 +0000 Subject: [PATCH 09/28] =?UTF-8?q?Add=20hubspot=5Fdeal=5Fid=20required=20fi?= =?UTF-8?q?eld=20to=20MagicPlanTriggerRequest=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../magic_plan/tests/test_magic_plan_trigger_request.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/magic_plan/tests/test_magic_plan_trigger_request.py b/backend/magic_plan/tests/test_magic_plan_trigger_request.py index 46a20a37..131ea93b 100644 --- a/backend/magic_plan/tests/test_magic_plan_trigger_request.py +++ b/backend/magic_plan/tests/test_magic_plan_trigger_request.py @@ -38,3 +38,11 @@ def test_extra_fields_ignored() -> None: req = MagicPlanTriggerRequest.model_validate(payload) # Assert assert req.address == "123 High St London SW1A 1AA" + + +def test_missing_hubspot_deal_id_raises() -> None: + # Arrange + payload = {"address": "123 High St London SW1A 1AA"} + # Act / Assert + with pytest.raises(ValidationError): + MagicPlanTriggerRequest.model_validate(payload) From 4a9cabe1979a1f0e12ef97f13dc634d4928b4fef Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:45:10 +0000 Subject: [PATCH 10/28] =?UTF-8?q?Add=20hubspot=5Fdeal=5Fid=20required=20fi?= =?UTF-8?q?eld=20to=20MagicPlanTriggerRequest=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- backend/magic_plan/handler.py | 2 +- backend/magic_plan/magic_plan_trigger_request.py | 1 + .../magic_plan/tests/test_magic_plan_trigger_request.py | 9 +++++---- etl/hubspot/scripts/scraper/main.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index a592cc6a..22933e13 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -28,7 +28,7 @@ if __name__ == "__main__": event = { "Records": [ { - "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ"}', + "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ", "hubspot_deal_id": "local-test-deal"}', "messageId": "local-test", } ] diff --git a/backend/magic_plan/magic_plan_trigger_request.py b/backend/magic_plan/magic_plan_trigger_request.py index bb0151e4..e93c055c 100644 --- a/backend/magic_plan/magic_plan_trigger_request.py +++ b/backend/magic_plan/magic_plan_trigger_request.py @@ -7,4 +7,5 @@ class MagicPlanTriggerRequest(BaseModel): model_config = ConfigDict(extra="ignore") address: str + hubspot_deal_id: str uprn: Optional[str] = None diff --git a/backend/magic_plan/tests/test_magic_plan_trigger_request.py b/backend/magic_plan/tests/test_magic_plan_trigger_request.py index 131ea93b..9fb2754a 100644 --- a/backend/magic_plan/tests/test_magic_plan_trigger_request.py +++ b/backend/magic_plan/tests/test_magic_plan_trigger_request.py @@ -6,17 +6,18 @@ from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerReques def test_valid_payload_with_address_only() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert assert req.address == "123 High St London SW1A 1AA" + assert req.hubspot_deal_id == "123456789" assert req.uprn is None def test_valid_payload_with_uprn() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA", "uprn": "100023336956"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789", "uprn": "100023336956"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert @@ -25,7 +26,7 @@ def test_valid_payload_with_uprn() -> None: def test_missing_address_raises() -> None: # Arrange - payload = {"uprn": "100023336956"} + payload = {"hubspot_deal_id": "123456789", "uprn": "100023336956"} # Act / Assert with pytest.raises(ValidationError): MagicPlanTriggerRequest.model_validate(payload) @@ -33,7 +34,7 @@ def test_missing_address_raises() -> None: def test_extra_fields_ignored() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA", "unknown_field": "whatever"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789", "unknown_field": "whatever"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index a39e8b37..32007cd4 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -62,7 +62,7 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) - _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing, hubspot_deal_id) else: # Deal already in db, check whether anything has changed logger.info( @@ -110,16 +110,17 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) - _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing, hubspot_deal_id) print("done") def _trigger_magicplan_fetcher( - sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] + sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]], hubspot_deal_id: str ) -> None: message_body = { "address": hubspot_deal.get("dealname"), + "hubspot_deal_id": hubspot_deal_id, "uprn": listing.get("national_uprn") if listing else None, } response = sqs_client.send_message( From c3aae8fd51373a9b38cbdb023275bbd12172519c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:08:56 +0000 Subject: [PATCH 11/28] =?UTF-8?q?Expose=20get=5Fplan=5Fraw=20method=20on?= =?UTF-8?q?=20MagicPlanClient=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_client.py | 3 ++ .../tests/test_magic_plan_client.py | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/backend/magic_plan/magic_plan_client.py b/backend/magic_plan/magic_plan_client.py index 60f70fb1..172190fd 100644 --- a/backend/magic_plan/magic_plan_client.py +++ b/backend/magic_plan/magic_plan_client.py @@ -22,3 +22,6 @@ class MagicPlanClient: ) r.raise_for_status() return MagicPlanPlan.model_validate(r.json()["data"]) + + def get_plan_raw(self, plan_id: str) -> bytes: + raise NotImplementedError diff --git a/backend/magic_plan/tests/test_magic_plan_client.py b/backend/magic_plan/tests/test_magic_plan_client.py index 1be1448f..c96b9cdf 100644 --- a/backend/magic_plan/tests/test_magic_plan_client.py +++ b/backend/magic_plan/tests/test_magic_plan_client.py @@ -172,3 +172,55 @@ def test_get_plan_propagates_http_error( # Act / Assert with pytest.raises(requests.HTTPError): client.get_plan("some-id") + + +# --- get_plan_raw --- + + +def test_get_plan_raw_returns_bytes( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b'{"data": "raw"}' + plan_id = "a7285ed1-878d-47eb-8aa6-85ef9e187516" + # Act + result = client.get_plan_raw(plan_id) + # Assert + assert isinstance(result, bytes) + + +def test_get_plan_raw_calls_correct_url( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b"{}" + plan_id = "a7285ed1-878d-47eb-8aa6-85ef9e187516" + # Act + client.get_plan_raw(plan_id) + # Assert + mock_session.get.assert_called_once_with( + f"{BASE_URL}/plans/{plan_id}", params={"key": API_KEY} + ) + + +def test_get_plan_raw_calls_raise_for_status( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b"{}" + # Act + client.get_plan_raw("a7285ed1-878d-47eb-8aa6-85ef9e187516") + # Assert + mock_session.get.return_value.raise_for_status.assert_called_once() + + +def test_get_plan_raw_propagates_http_error( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.raise_for_status.side_effect = requests.HTTPError( + "500" + ) + # Act / Assert + with pytest.raises(requests.HTTPError): + client.get_plan_raw("some-id") From f6c17be70a3d85e0638cea58f3edcdaf530c0aee Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:09:33 +0000 Subject: [PATCH 12/28] =?UTF-8?q?Expose=20get=5Fplan=5Fraw=20method=20on?= =?UTF-8?q?=20MagicPlanClient=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/magic_plan/magic_plan_client.py b/backend/magic_plan/magic_plan_client.py index 172190fd..06905e6a 100644 --- a/backend/magic_plan/magic_plan_client.py +++ b/backend/magic_plan/magic_plan_client.py @@ -17,11 +17,14 @@ class MagicPlanClient: return PlansListResponse.model_validate(r.json()["data"]) def get_plan(self, plan_id: str) -> MagicPlanPlan: + return MagicPlanPlan.model_validate(self._fetch_plan(plan_id).json()["data"]) + + def get_plan_raw(self, plan_id: str) -> bytes: + return self._fetch_plan(plan_id).content + + def _fetch_plan(self, plan_id: str) -> requests.Response: r = self._session.get( f"{_BASE_URL}/plans/{plan_id}", params={"key": self._api_key} ) r.raise_for_status() - return MagicPlanPlan.model_validate(r.json()["data"]) - - def get_plan_raw(self, plan_id: str) -> bytes: - raise NotImplementedError + return r From 7c9cb5b161b4e910d95c96c6633362000c4b9e18 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:14:42 +0000 Subject: [PATCH 13/28] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/db/models/uploaded_file.py | 2 + backend/magic_plan/handler.py | 2 +- backend/magic_plan/magic_plan_service.py | 13 +++- .../tests/test_magic_plan_service.py | 66 +++++++++++++++++-- 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/backend/app/db/models/uploaded_file.py b/backend/app/db/models/uploaded_file.py index a516a1df..c629f574 100644 --- a/backend/app/db/models/uploaded_file.py +++ b/backend/app/db/models/uploaded_file.py @@ -17,6 +17,7 @@ class FileTypeEnum(enum.Enum): ECMK_SITE_NOTE = "ecmk_site_note" ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" ECMK_SURVEY_XML = "ecmk_survey_xml" + MAGIC_PLAN_JSON = "magic_plan_json" class FileSourceEnum(enum.Enum): @@ -24,6 +25,7 @@ class FileSourceEnum(enum.Enum): SHAREPOINT = "sharepoint" HUBSPOT = "hubspot" ECMK = "ecmk" + MAGIC_PLAN = "magic_plan" class UploadedFile(Base): diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index 22933e13..45de8554 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -19,7 +19,7 @@ def handler(body: dict[str, Any], context: Any) -> str: customer_id=settings.MAGICPLAN_CUSTOMER_ID, api_key=settings.MAGICPLAN_API_KEY, ) - plan: Plan = MagicPlanService(client).run(payload.address, payload.uprn) + plan: Plan = MagicPlanService(client, s3_bucket="retrofit-energy-assessments-dev").run(payload) logger.info("Saved MagicPlan plan uid=%s", plan.uid) return plan.uid diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 91b3cd13..6be6486c 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,3 +1,4 @@ +import gzip from typing import Optional from datatypes.magicplan.api.response import ( @@ -12,23 +13,29 @@ from backend.app.db.connection import db_session from backend.app.db.functions.magic_plan_functions import save_plan from backend.magic_plan.address_matcher import find_matching_plan from backend.magic_plan.magic_plan_client import MagicPlanClient +from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest from utils.logger import setup_logger +from utils.s3 import save_data_to_s3 logger = setup_logger() class MagicPlanService: - def __init__(self, client: MagicPlanClient) -> None: + def __init__(self, client: MagicPlanClient, s3_bucket: str) -> None: self._client = client + self._s3_bucket = s3_bucket + + def run(self, request: MagicPlanTriggerRequest) -> Plan: + address = request.address + uprn = request.uprn - def run(self, address: str, uprn: Optional[str] = None) -> Plan: if uprn is not None: logger.info("MagicPlanService.run uprn=%s", uprn) plans_response: PlansListResponse = self._client.get_plans() matched: Optional[PlanSummary] = find_matching_plan( plans_response.plans, address - ) # TODO: use address2UPRN instead? or create AddressMatch domain class + ) if matched is None: raise ValueError(f"No MagicPlan found for address: {address!r}") diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 8e433b87..87e20506 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch import pytest @@ -10,9 +10,11 @@ from datatypes.magicplan.domain.models import Plan from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_service import MagicPlanService +from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest FIXTURE_DIR = Path(__file__).parents[2] / "magic_plan" PLAN_ID = "a7285ed1-878d-47eb-8aa6-85ef9e187516" +S3_BUCKET = "test-bucket" @pytest.fixture(scope="module") @@ -45,7 +47,17 @@ def mock_client() -> MagicMock: def _make_service(mock_client: MagicMock) -> MagicPlanService: - return MagicPlanService(client=mock_client) + return MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + + +def _make_request( + address: str = "2 Laburnum Way Bromley BR2 8BZ", + hubspot_deal_id: str = "deal-123", + uprn: str | None = None, +) -> MagicPlanTriggerRequest: + return MagicPlanTriggerRequest( + address=address, hubspot_deal_id=hubspot_deal_id, uprn=uprn + ) # --- no match --- @@ -57,7 +69,7 @@ def test_run_raises_when_no_plan_found(mock_client: MagicMock) -> None: service = _make_service(mock_client) # Act / Assert with pytest.raises(ValueError, match="No MagicPlan found"): - service.run("99 Nowhere Road London SW1A 1AA") + service.run(_make_request(address="99 Nowhere Road London SW1A 1AA")) # --- match found --- @@ -78,8 +90,10 @@ def test_run_fetches_plan_with_matched_id( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ") + service.run(_make_request()) # Assert mock_client.get_plan.assert_called_once_with(plan_summary.id) @@ -99,8 +113,10 @@ def test_run_returns_mapped_plan( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - result = service.run("2 Laburnum Way Bromley BR2 8BZ") + result = service.run(_make_request()) # Assert assert isinstance(result, Plan) assert result.uid == PLAN_ID @@ -120,8 +136,10 @@ def test_run_calls_save_plan_with_mapped_plan( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan") as mock_save, patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ") + service.run(_make_request()) # Assert — save_plan called with a Plan whose uid matches call_args = mock_save.call_args saved_plan: Plan = call_args[0][1] @@ -142,5 +160,39 @@ def test_run_accepts_uprn_without_error( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ", uprn="100023336956") + service.run(_make_request(uprn="100023336956")) + + +# --- S3 upload --- + + +def test_run_uploads_to_s3_with_uprn_key( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + mock_client.get_plan_raw.return_value = b'{"raw": "data"}' + request = _make_request(uprn="100023336956") + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ) as mock_s3: + # Act + service.run(request) + # Assert + mock_s3.assert_called_once_with( + ANY, + S3_BUCKET, + f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz", + ) From 14a064fdefd1d6a9362cf3a4c078bbfc6da5b316 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:16:41 +0000 Subject: [PATCH 14/28] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 8 ++++++++ backend/magic_plan/tests/test_magic_plan_service.py | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 6be6486c..bb68fa42 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -41,8 +41,16 @@ class MagicPlanService: raise ValueError(f"No MagicPlan found for address: {address!r}") magic_plan: MagicPlanPlan = self._client.get_plan(matched.id) + raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) + compressed = gzip.compress(raw_bytes) + if uprn is not None: + s3_key = f"documents/uprn/{uprn}/magic_plan_{matched.id}.json.gz" + else: + s3_key = f"documents/hubspot_deal_id/{request.hubspot_deal_id}/magic_plan_{matched.id}.json.gz" + save_data_to_s3(compressed, self._s3_bucket, s3_key) + with db_session() as session: save_plan(session, plan) diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 87e20506..65c19b7a 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -43,7 +43,9 @@ def plan_summary() -> PlanSummary: @pytest.fixture() def mock_client() -> MagicMock: - return MagicMock(spec=MagicPlanClient) + client = MagicMock(spec=MagicPlanClient) + client.get_plan_raw.return_value = b"{}" + return client def _make_service(mock_client: MagicMock) -> MagicPlanService: From 03e8750c1ac233b408820eb56b4976b7b5a97d0b Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:17:51 +0000 Subject: [PATCH 15/28] =?UTF-8?q?Upload=20MagicPlan=20JSON=20to=20S3=20usi?= =?UTF-8?q?ng=20hubspot=5Fdeal=5Fid=20key=20when=20UPRN=20absent=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_magic_plan_service.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 65c19b7a..70099e91 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -198,3 +198,31 @@ def test_run_uploads_to_s3_with_uprn_key( S3_BUCKET, f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz", ) + + +def test_run_uploads_to_s3_with_deal_id_key_when_uprn_absent( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + request = _make_request(hubspot_deal_id="deal-456", uprn=None) + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ) as mock_s3: + # Act + service.run(request) + # Assert + mock_s3.assert_called_once_with( + ANY, + S3_BUCKET, + f"documents/hubspot_deal_id/deal-456/magic_plan_{plan_summary.id}.json.gz", + ) From 8ac77ce8b967eb87884443ad4e40e5e2d8c6fa29 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:19:49 +0000 Subject: [PATCH 16/28] =?UTF-8?q?Persist=20UploadedFile=20record=20for=20e?= =?UTF-8?q?ach=20MagicPlan=20S3=20upload=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 5 +++ .../tests/test_magic_plan_service.py | 45 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index bb68fa42..0aeed686 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -11,6 +11,11 @@ from datatypes.magicplan.domain.models import Plan from backend.app.db.connection import db_session from backend.app.db.functions.magic_plan_functions import save_plan +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) from backend.magic_plan.address_matcher import find_matching_plan from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 70099e91..b7580546 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -8,6 +8,11 @@ from datatypes.magicplan.api.response import MagicPlanPlan, PlanSummary from datatypes.magicplan.domain.mapper import map_plan from datatypes.magicplan.domain.models import Plan +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_service import MagicPlanService from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest @@ -226,3 +231,43 @@ def test_run_uploads_to_s3_with_deal_id_key_when_uprn_absent( S3_BUCKET, f"documents/hubspot_deal_id/deal-456/magic_plan_{plan_summary.id}.json.gz", ) + + +# --- UploadedFile record --- + + +def test_run_creates_uploaded_file_record( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + request = _make_request(hubspot_deal_id="deal-789", uprn="100023336956") + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + mock_session = MagicMock() + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ) as mock_db, patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ): + mock_db.return_value.__enter__.return_value = mock_session + # Act + service.run(request) + # Assert + added_objects = [call.args[0] for call in mock_session.add.call_args_list] + uploaded_file = next( + (obj for obj in added_objects if isinstance(obj, UploadedFile)), None + ) + assert uploaded_file is not None + assert uploaded_file.file_source == FileSourceEnum.MAGIC_PLAN.value + assert uploaded_file.file_type == FileTypeEnum.MAGIC_PLAN_JSON.value + assert uploaded_file.s3_file_bucket == S3_BUCKET + assert uploaded_file.s3_file_key == f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz" + assert uploaded_file.s3_upload_timestamp is not None + assert uploaded_file.uprn == 100023336956 + assert uploaded_file.hubspot_deal_id == "deal-789" From 337474e7733110e633601c5a7eaed26d6bbe4241 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:20:49 +0000 Subject: [PATCH 17/28] =?UTF-8?q?Persist=20UploadedFile=20record=20for=20e?= =?UTF-8?q?ach=20MagicPlan=20S3=20upload=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 0aeed686..7860fec9 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,4 +1,5 @@ import gzip +from datetime import datetime, timezone from typing import Optional from datatypes.magicplan.api.response import ( @@ -58,5 +59,16 @@ class MagicPlanService: with db_session() as session: save_plan(session, plan) + session.add( + UploadedFile( + s3_file_bucket=self._s3_bucket, + s3_file_key=s3_key, + s3_upload_timestamp=datetime.now(timezone.utc), + uprn=int(uprn) if uprn is not None else None, + hubspot_deal_id=request.hubspot_deal_id, + file_source=FileSourceEnum.MAGIC_PLAN.value, + file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, + ) + ) return plan From e1972e4349d86db57eca6ad5230bcc900a10935a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:23:32 +0000 Subject: [PATCH 18/28] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 47 +++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 7860fec9..9ac17e55 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -50,25 +50,38 @@ class MagicPlanService: raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) - compressed = gzip.compress(raw_bytes) - if uprn is not None: - s3_key = f"documents/uprn/{uprn}/magic_plan_{matched.id}.json.gz" - else: - s3_key = f"documents/hubspot_deal_id/{request.hubspot_deal_id}/magic_plan_{matched.id}.json.gz" - save_data_to_s3(compressed, self._s3_bucket, s3_key) + uploaded_file = self._upload_raw_plan_json( + plan_id=matched.id, + raw_bytes=raw_bytes, + uprn=uprn, + hubspot_deal_id=request.hubspot_deal_id, + ) with db_session() as session: save_plan(session, plan) - session.add( - UploadedFile( - s3_file_bucket=self._s3_bucket, - s3_file_key=s3_key, - s3_upload_timestamp=datetime.now(timezone.utc), - uprn=int(uprn) if uprn is not None else None, - hubspot_deal_id=request.hubspot_deal_id, - file_source=FileSourceEnum.MAGIC_PLAN.value, - file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, - ) - ) + session.add(uploaded_file) return plan + + def _upload_raw_plan_json( + self, + plan_id: str, + raw_bytes: bytes, + uprn: Optional[str], + hubspot_deal_id: str, + ) -> UploadedFile: + compressed = gzip.compress(raw_bytes) + if uprn is not None: + s3_key = f"documents/uprn/{uprn}/magic_plan_{plan_id}.json.gz" + else: + s3_key = f"documents/hubspot_deal_id/{hubspot_deal_id}/magic_plan_{plan_id}.json.gz" + save_data_to_s3(compressed, self._s3_bucket, s3_key) + return UploadedFile( + s3_file_bucket=self._s3_bucket, + s3_file_key=s3_key, + s3_upload_timestamp=datetime.now(timezone.utc), + uprn=int(uprn) if uprn is not None else None, + hubspot_deal_id=hubspot_deal_id, + file_source=FileSourceEnum.MAGIC_PLAN.value, + file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, + ) From 9f62e3c31abe61a9f67faf38e3bbc77730a1b64d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:30:59 +0000 Subject: [PATCH 19/28] typehint --- backend/magic_plan/magic_plan_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 9ac17e55..6ed25c0c 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -50,7 +50,7 @@ class MagicPlanService: raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) - uploaded_file = self._upload_raw_plan_json( + uploaded_file: UploadedFile = self._upload_raw_plan_json( plan_id=matched.id, raw_bytes=raw_bytes, uprn=uprn, From ce2b61d60b10b5608f390a267ad2fe2d92b7d164 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:07:09 +0000 Subject: [PATCH 20/28] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20-=20only=20make=20one=20API=20call=20?= =?UTF-8?q?=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 5 ++++- backend/magic_plan/tests/test_magic_plan_service.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 6ed25c0c..fb0a7610 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,4 +1,5 @@ import gzip +import json from datetime import datetime, timezone from typing import Optional @@ -46,8 +47,10 @@ class MagicPlanService: if matched is None: raise ValueError(f"No MagicPlan found for address: {address!r}") - magic_plan: MagicPlanPlan = self._client.get_plan(matched.id) raw_bytes: bytes = self._client.get_plan_raw(matched.id) + magic_plan: MagicPlanPlan = MagicPlanPlan.model_validate( + json.loads(raw_bytes)["data"] + ) plan: Plan = map_plan(magic_plan) uploaded_file: UploadedFile = self._upload_raw_plan_json( diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index b7580546..f6954824 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -49,7 +49,9 @@ def plan_summary() -> PlanSummary: @pytest.fixture() def mock_client() -> MagicMock: client = MagicMock(spec=MagicPlanClient) - client.get_plan_raw.return_value = b"{}" + client.get_plan_raw.return_value = ( + FIXTURE_DIR / "magicplan_api_plan_response_example.json" + ).read_bytes() return client @@ -102,7 +104,7 @@ def test_run_fetches_plan_with_matched_id( ): service.run(_make_request()) # Assert - mock_client.get_plan.assert_called_once_with(plan_summary.id) + mock_client.get_plan_raw.assert_called_once_with(plan_summary.id) def test_run_returns_mapped_plan( @@ -183,8 +185,6 @@ def test_run_uploads_to_s3_with_uprn_key( ) -> None: # Arrange mock_client.get_plans.return_value.plans = [plan_summary] - mock_client.get_plan.return_value = api_magic_plan - mock_client.get_plan_raw.return_value = b'{"raw": "data"}' request = _make_request(uprn="100023336956") service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) with patch( From 1243690d100a0d73b6a91100e02ed289f160b87a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:41:12 +0000 Subject: [PATCH 21/28] give handler permission to write to s3 bucket --- backend/magic_plan/handler.py | 1 + infrastructure/terraform/lambda/magic_plan/main.tf | 5 +++++ infrastructure/terraform/shared/main.tf | 14 ++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index 45de8554..f2c03b90 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -19,6 +19,7 @@ def handler(body: dict[str, Any], context: Any) -> str: customer_id=settings.MAGICPLAN_CUSTOMER_ID, api_key=settings.MAGICPLAN_API_KEY, ) + # TODO: read s3_bucket from env var so staging/prod use the correct bucket plan: Plan = MagicPlanService(client, s3_bucket="retrofit-energy-assessments-dev").run(payload) logger.info("Saved MagicPlan plan uid=%s", plan.uid) return plan.uid diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf index 56adac1b..e2017b42 100644 --- a/infrastructure/terraform/lambda/magic_plan/main.tf +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -15,6 +15,11 @@ locals { db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) } +resource "aws_iam_role_policy_attachment" "magic_plan_s3_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.magic_plan_s3_write_arn +} + module "lambda" { source = "../../modules/lambda_with_sqs" diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 050ebdc2..e32ce395 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -745,4 +745,18 @@ module "magic_plan_client_registry" { source = "../modules/container_registry" name = "magic-plan" stage = var.stage +} + +module "magic_plan_s3_write" { + source = "../modules/s3_iam_policy" + + policy_name = "MagicPlanWriteS3" + policy_description = "Allow MagicPlan Lambda to write to retrofit energy assessments bucket" + bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] + actions = ["s3:PutObject", "s3:AbortMultipartUpload"] + resource_paths = ["/*"] +} + +output "magic_plan_s3_write_arn" { + value = module.magic_plan_s3_write.policy_arn } \ No newline at end of file From aadf73ed87db510efc17a1579122d7bc5e65ca15 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:44:14 +0000 Subject: [PATCH 22/28] combine s3 write policies into one and apply to pashub and magicplan lambdas --- .../terraform/lambda/magic_plan/main.tf | 2 +- .../terraform/lambda/pashub_to_ara/main.tf | 2 +- infrastructure/terraform/shared/main.tf | 23 ++++--------------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf index e2017b42..48fc3867 100644 --- a/infrastructure/terraform/lambda/magic_plan/main.tf +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -17,7 +17,7 @@ locals { resource "aws_iam_role_policy_attachment" "magic_plan_s3_write" { role = module.lambda.role_name - policy_arn = data.terraform_remote_state.shared.outputs.magic_plan_s3_write_arn + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn } module "lambda" { diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/infrastructure/terraform/lambda/pashub_to_ara/main.tf index 1a457617..902d7845 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/main.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/main.tf @@ -54,5 +54,5 @@ module "lambda" { resource "aws_iam_role_policy_attachment" "pashub_to_ara_s3_write" { role = module.lambda.role_name - policy_arn = data.terraform_remote_state.shared.outputs.pashub_to_ara_s3_write_arn + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn } diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index e32ce395..2c3200de 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -568,18 +568,18 @@ module "pashub_to_ara_registry" { stage = var.stage } -module "pashub_to_ara_s3_write" { +module "energy_assessments_s3_write" { source = "../modules/s3_iam_policy" - policy_name = "PashubToAraWriteS3" - policy_description = "Allow PasHub to ARA Lambda to write to retrofit energy assessments bucket" + policy_name = "EnergyAssessmentsWriteS3" + policy_description = "Allow lambdas to write to retrofit energy assessments bucket" bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] actions = ["s3:PutObject", "s3:AbortMultipartUpload"] resource_paths = ["/*"] } -output "pashub_to_ara_s3_write_arn" { - value = module.pashub_to_ara_s3_write.policy_arn +output "energy_assessments_s3_write_arn" { + value = module.energy_assessments_s3_write.policy_arn } ################################################ @@ -747,16 +747,3 @@ module "magic_plan_client_registry" { stage = var.stage } -module "magic_plan_s3_write" { - source = "../modules/s3_iam_policy" - - policy_name = "MagicPlanWriteS3" - policy_description = "Allow MagicPlan Lambda to write to retrofit energy assessments bucket" - bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] - actions = ["s3:PutObject", "s3:AbortMultipartUpload"] - resource_paths = ["/*"] -} - -output "magic_plan_s3_write_arn" { - value = module.magic_plan_s3_write.policy_arn -} \ No newline at end of file From 2049553176ee88e1fefa2062f300d036e58206a1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 11 May 2026 09:25:41 +0000 Subject: [PATCH 23/28] =?UTF-8?q?Trigger=20MagicPlan=20on=20outcome=20"sur?= =?UTF-8?q?veyed"=20transition=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 7 +-- etl/hubspot/tests/test_hubspot_deal_differ.py | 49 ++++++++++--------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 5435a46d..ba3dc27a 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -198,12 +198,7 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - new_status = (new_deal.get("coordination_status__stage_1_") or "").lower() - old_status = (old_deal.coordination_status or "").lower() - return ( - new_status in HubspotDealDiffer.COORDINATION_COMPLETE - and old_status not in HubspotDealDiffer.COORDINATION_COMPLETE - ) + raise NotImplementedError @staticmethod def _lodgement_completed( diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 273a82a0..94952424 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -275,15 +275,12 @@ def test_pashub_trigger__coordination_design_lodgement_not_completed_and_pashub_ # ========================== -def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() -> None: +def test_magicplan_trigger__outcome_transitions_to_surveyed__returns_true() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal(id=deal_id, coordination_status="in progress") - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, - ) + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="surveyed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -295,20 +292,12 @@ def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() assert result is True -def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__returns_false() -> None: +def test_magicplan_trigger__outcome_already_surveyed__returns_false() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal( - id=deal_id, - coordination_status="(v1) ioe/mtp complete", - outcome="pending", - ) - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, - outcome="won", - ) + old_deal = make_old_deal(id=deal_id, outcome="surveyed") + new_deal = make_new_deal(deal_id, outcome="surveyed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -320,15 +309,12 @@ def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__r assert result is False -def test_magicplan_trigger__transitions_to_non_complete_coordination_status__returns_false() -> None: +def test_magicplan_trigger__outcome_transitions_to_non_surveyed__returns_false() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal(id=deal_id, coordination_status="in progress") - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "design submitted"}, - ) + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="assessed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -340,6 +326,23 @@ def test_magicplan_trigger__transitions_to_non_complete_coordination_status__ret assert result is False +def test_magicplan_trigger__outcome_surveyed_uppercase__returns_true() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="SURVEYED") + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is True + + # ======================= # DB UPDATE TRIGGER TESTS # ======================= From c15ffdf2c01765b0baa3e1fb371afe8c54e462c4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 11 May 2026 09:26:20 +0000 Subject: [PATCH 24/28] =?UTF-8?q?Trigger=20MagicPlan=20on=20outcome=20"sur?= =?UTF-8?q?veyed"=20transition=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index ba3dc27a..724a3e68 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -198,7 +198,9 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - raise NotImplementedError + new_outcome = (new_deal.get("outcome") or "").lower() + old_outcome = (old_deal.outcome or "").lower() + return new_outcome == "surveyed" and old_outcome != "surveyed" @staticmethod def _lodgement_completed( From f0300eb8ff4749da93a49a259c88f448ab7dee08 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:57:24 +0000 Subject: [PATCH 25/28] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 16 ++--- .../hubspot/tests/test_scraper_handler.py | 65 +++++++++---------- pytest.ini | 2 +- 3 files changed, 39 insertions(+), 44 deletions(-) rename backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py => etl/hubspot/tests/test_scraper_handler.py (61%) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 724a3e68..da0072c1 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -162,6 +162,14 @@ class HubspotDealDiffer: return False + @staticmethod + def check_for_magicplan_trigger( + new_deal: Dict[str, str], old_deal: HubspotDealData + ) -> bool: + new_outcome = (new_deal.get("outcome") or "").lower() + old_outcome = (old_deal.outcome or "").lower() + return new_outcome == "surveyed" and old_outcome != "surveyed" + @staticmethod def _has_valid_pashub_link(new_pashub_link: str) -> bool: return bool(new_pashub_link) @@ -194,14 +202,6 @@ class HubspotDealDiffer: and new_status != old_deal.design_status ) - @staticmethod - def check_for_magicplan_trigger( - new_deal: Dict[str, str], old_deal: HubspotDealData - ) -> bool: - new_outcome = (new_deal.get("outcome") or "").lower() - old_outcome = (old_deal.outcome or "").lower() - return new_outcome == "surveyed" and old_outcome != "surveyed" - @staticmethod def _lodgement_completed( new_deal: Dict[str, str], old_deal: HubspotDealData diff --git a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py b/etl/hubspot/tests/test_scraper_handler.py similarity index 61% rename from backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py rename to etl/hubspot/tests/test_scraper_handler.py index 6d18c4b4..e2f80d07 100644 --- a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -3,37 +3,28 @@ import uuid from typing import Any, Dict, Optional from unittest.mock import MagicMock, patch -import pytest - from backend.app.db.models.hubspot_deal_data import HubspotDealData from etl.hubspot.scripts.scraper.main import handler -COORDINATION_COMPLETE = "(v1) ioe/mtp complete" DEAL_NAME = "123 Main Street" UPRN = "12345678" DEAL_ID = "999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" -def make_hubspot_deal( - coordination_status: Optional[str] = None, **kwargs: Any -) -> Dict[str, Any]: - deal: Dict[str, Any] = { +def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: + return { "hs_object_id": DEAL_ID, "dealname": DEAL_NAME, "pashub_link": None, **kwargs, } - if coordination_status is not None: - deal["coordination_status__stage_1_"] = coordination_status - return deal -def make_db_deal(coordination_status: Optional[str] = None, **kwargs: Any) -> HubspotDealData: +def make_db_deal(**kwargs: Any) -> HubspotDealData: return HubspotDealData( id=uuid.uuid4(), deal_id=DEAL_ID, - coordination_status=coordination_status, **kwargs, ) @@ -68,14 +59,14 @@ def run_handler( return mock_sqs -# ======================= -# NEW DEAL PATH -# ======================= +# ==================================== +# NEW DEAL PATH - MagicPlan trigger +# ==================================== -def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: +def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} # Act @@ -84,13 +75,15 @@ def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": UPRN} + ), ) -def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: +def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status="in progress") + hubspot_deal = make_hubspot_deal(outcome="assessed") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) @@ -99,9 +92,9 @@ def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: mock_sqs.send_message.assert_not_called() -def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: +def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_null() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + hubspot_deal = make_hubspot_deal(outcome="surveyed") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) @@ -109,19 +102,21 @@ def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": None}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": None} + ), ) -# ======================= -# EXISTING DEAL PATH -# ======================= +# ========================================== +# EXISTING DEAL PATH - MagicPlan trigger +# ========================================== -def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() -> None: +def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> None: # Arrange - db_deal = make_db_deal(coordination_status="in progress") - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + db_deal = make_db_deal(outcome="assessed") + hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} # Act @@ -130,16 +125,16 @@ def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": UPRN} + ), ) -def test_existing_deal_already_in_coordination_complete_unrelated_change__no_sqs_message() -> None: +def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_trigger_magicplan() -> None: # Arrange - db_deal = make_db_deal(coordination_status=COORDINATION_COMPLETE, dealname="Old Name") - hubspot_deal = make_hubspot_deal( - coordination_status=COORDINATION_COMPLETE, dealname="New Name" - ) + db_deal = make_db_deal(outcome="surveyed", dealname="Old Name") + hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) diff --git a/pytest.ini b/pytest.ini index 398c5b71..e2a4a25d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,6 +3,6 @@ pythonpath = . log_cli = true log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial -testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests etl/hubspot/tests backend/hubspot_trigger_orchestrator/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests datatypes/epc/domain/tests backend/ecmk_fetcher/tests/ backend/pashub_fetcher/tests backend/documents_parser/tests backend/magic_plan/tests datatypes/magicplan/api/tests datatypes/magicplan/domain/tests backend/app/db/functions/tests +testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests etl/hubspot/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests datatypes/epc/domain/tests backend/ecmk_fetcher/tests/ backend/pashub_fetcher/tests backend/documents_parser/tests backend/magic_plan/tests datatypes/magicplan/api/tests datatypes/magicplan/domain/tests backend/app/db/functions/tests markers = integration: mark a test as an integration test From 9386846044e087b38a15bbfae3dd306f376cd205 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:58:15 +0000 Subject: [PATCH 26/28] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/scripts/scraper/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 32007cd4..86844352 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -57,8 +57,7 @@ def handler(body: dict[str, Any], context: Any) -> None: ) _trigger_pashub_fetcher(sqs_client, hubspot_deal_id, hubspot_deal) - coordination_status = (hubspot_deal.get("coordination_status__stage_1_") or "").lower() - if coordination_status in HubspotDealDiffer.COORDINATION_COMPLETE: + if (hubspot_deal.get("outcome") or "").lower() == "surveyed": logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) From 9501146ec815b2f2998017acb646571c922ba277 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:59:32 +0000 Subject: [PATCH 27/28] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/hubspot_trigger_orchestrator/__init__.py | 0 backend/hubspot_trigger_orchestrator/tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/hubspot_trigger_orchestrator/__init__.py delete mode 100644 backend/hubspot_trigger_orchestrator/tests/__init__.py diff --git a/backend/hubspot_trigger_orchestrator/__init__.py b/backend/hubspot_trigger_orchestrator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/hubspot_trigger_orchestrator/tests/__init__.py b/backend/hubspot_trigger_orchestrator/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 From 71aadfe78d1237cca0721707d97b7fc01e6bb6c3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 09:31:47 +0000 Subject: [PATCH 28/28] add pashub functions to orchestrator tests, and rename existing magicplan ones --- etl/hubspot/tests/test_scraper_handler.py | 96 +++++++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index e2f80d07..4810d171 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -9,7 +9,9 @@ from etl.hubspot.scripts.scraper.main import handler DEAL_NAME = "123 Main Street" UPRN = "12345678" DEAL_ID = "999" +PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" +PASHUB_QUEUE_URL = "https://sqs.test/pashub" def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: @@ -52,7 +54,7 @@ def run_handler( ) mock_boto3.client.return_value = mock_sqs mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL - mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = "https://sqs.test/pashub" + mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = PASHUB_QUEUE_URL handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") @@ -64,7 +66,7 @@ def run_handler( # ==================================== -def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: +def test_new_deal_surveyed__sends_magicplan_sqs() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} @@ -81,7 +83,7 @@ def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: ) -def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None: +def test_new_deal_not_surveyed__no_magicplan_sqs() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="assessed") @@ -92,7 +94,7 @@ def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None mock_sqs.send_message.assert_not_called() -def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_null() -> None: +def test_new_deal_surveyed_no_listing__magicplan_sqs_uprn_is_null() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="surveyed") @@ -113,7 +115,7 @@ def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_nu # ========================================== -def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> None: +def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None: # Arrange db_deal = make_db_deal(outcome="assessed") hubspot_deal = make_hubspot_deal(outcome="surveyed") @@ -131,7 +133,7 @@ def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> ) -def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_trigger_magicplan() -> None: +def test_existing_deal_already_surveyed__no_magicplan_sqs() -> None: # Arrange db_deal = make_db_deal(outcome="surveyed", dealname="Old Name") hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") @@ -141,3 +143,85 @@ def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_tri # Assert mock_sqs.send_message.assert_not_called() + + +# ==================================== +# NEW DEAL PATH - PasHub trigger +# ==================================== + + +def test_new_deal_with_pashub_link__sends_pashub_sqs() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=PASHUB_QUEUE_URL, + MessageBody=json.dumps( + { + "pashub_link": PASHUB_LINK, + "address": None, + "hubspot_deal_id": DEAL_ID, + "sharepoint_link": None, + "uprn": None, + "landlord_property_id": None, + "deal_stage": None, + } + ), + ) + + +def test_new_deal_no_pashub_link__no_pashub_sqs() -> None: + # Arrange + hubspot_deal = make_hubspot_deal() + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() + + +# ========================================== +# EXISTING DEAL PATH - PasHub trigger +# ========================================== + + +def test_existing_deal_pashub_link_added__sends_pashub_sqs() -> None: + # Arrange + db_deal = make_db_deal(pashub_link=None) + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=PASHUB_QUEUE_URL, + MessageBody=json.dumps( + { + "pashub_link": PASHUB_LINK, + "address": None, + "hubspot_deal_id": DEAL_ID, + "sharepoint_link": None, + "uprn": None, + "landlord_property_id": None, + "deal_stage": None, + } + ), + ) + + +def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: + # Arrange + db_deal = make_db_deal(pashub_link=PASHUB_LINK, dealname="Old Name") + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name") + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called()