From 3683d9141f694fe3f51aa1e8ea1b8cb61be493c7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Apr 2026 15:04:10 +0100 Subject: [PATCH 01/32] protecting pushes to dev --- .github/workflows/protect_releases.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/protect_releases.yml diff --git a/.github/workflows/protect_releases.yml b/.github/workflows/protect_releases.yml new file mode 100644 index 00000000..cbd08e2f --- /dev/null +++ b/.github/workflows/protect_releases.yml @@ -0,0 +1,17 @@ +name: Restrict PR source + +on: + pull_request: + branches: + - dev + +jobs: + check-source-branch: + runs-on: ubuntu-latest + steps: + - name: Fail if PR is not from main + run: | + if [[ "${{ github.head_ref }}" != "main" ]]; then + echo "Only PRs from main are allowed into dev" + exit 1 + fi \ No newline at end of file From 676022a4c054d7301b1d7542b582fadbb63705b4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 12:53:37 +0000 Subject: [PATCH 02/32] =?UTF-8?q?Fix=20coordination/design=20field=20names?= =?UTF-8?q?=20and=20add=20MagicPlan=20trigger=20to=20HubspotDealDiffer=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 6 ++ etl/hubspot/tests/test_hubspot_deal_differ.py | 76 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 9e7069fc..a53df4f7 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -194,6 +194,12 @@ class HubspotDealDiffer: and new_status != old_deal.design_status ) + @staticmethod + def check_for_magicplan_trigger( + new_deal: Dict[str, str], old_deal: HubspotDealData + ) -> bool: + raise NotImplementedError + @staticmethod def _lodgement_completed( new_deal: Dict[str, str], old_deal: HubspotDealData diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 0523c982..273a82a0 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -109,7 +109,7 @@ def test_pashub_trigger__coordination_completed_and_pashub_link_set__returns_tru new_deal = make_new_deal( deal_id, pashub_link="www.google.co.uk", - coordination_status=coordination_status, + **{"coordination_status__stage_1_": coordination_status}, ) assert ( @@ -156,7 +156,7 @@ def test_pashub_trigger__design_completed_and_pashub_link_set__returns_true() -> new_deal = make_new_deal( deal_id, pashub_link="www.google.co.uk", - design_status="uploaded", + retrofit_design_status="uploaded", ) assert ( @@ -177,7 +177,7 @@ def test_pashub_trigger__design_completed_and_pashub_link_not_set__returns_false new_deal = make_new_deal( deal_id, - design_status="uploaded", + retrofit_design_status="uploaded", ) assert ( @@ -270,6 +270,76 @@ def test_pashub_trigger__coordination_design_lodgement_not_completed_and_pashub_ ) +# ========================== +# MAGICPLAN TRIGGER TESTS +# ========================== + + +def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, coordination_status="in progress") + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is True + + +def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__returns_false() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal( + id=deal_id, + coordination_status="(v1) ioe/mtp complete", + outcome="pending", + ) + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, + outcome="won", + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is False + + +def test_magicplan_trigger__transitions_to_non_complete_coordination_status__returns_false() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, coordination_status="in progress") + new_deal = make_new_deal( + deal_id, + **{"coordination_status__stage_1_": "design submitted"}, + ) + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is False + + # ======================= # DB UPDATE TRIGGER TESTS # ======================= From 69faa530a4c5a4dce34a00919baf965e2f59943a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 12:55:23 +0000 Subject: [PATCH 03/32] =?UTF-8?q?Fix=20coordination/design=20field=20names?= =?UTF-8?q?=20and=20add=20MagicPlan=20trigger=20to=20HubspotDealDiffer=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index a53df4f7..5435a46d 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -178,7 +178,7 @@ class HubspotDealDiffer: def _coordination_completed( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - new_status: str = new_deal.get("coordination_status") or "" + new_status: str = new_deal.get("coordination_status__stage_1_") or "" return ( new_status != "" and new_status.lower() in HubspotDealDiffer.COORDINATION_COMPLETE @@ -187,7 +187,7 @@ class HubspotDealDiffer: @staticmethod def _design_completed(new_deal: Dict[str, str], old_deal: HubspotDealData) -> bool: - new_status: str = new_deal.get("design_status") or "" + new_status: str = new_deal.get("retrofit_design_status") or "" return ( new_status != "" and new_status.lower() == HubspotDealDiffer.RETROFIT_DESIGN_COMPLETE @@ -198,7 +198,12 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - raise NotImplementedError + new_status = (new_deal.get("coordination_status__stage_1_") or "").lower() + old_status = (old_deal.coordination_status or "").lower() + return ( + new_status in HubspotDealDiffer.COORDINATION_COMPLETE + and old_status not in HubspotDealDiffer.COORDINATION_COMPLETE + ) @staticmethod def _lodgement_completed( From 489b0ba30eb730139916901da7b585627428c3e9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:05:38 +0000 Subject: [PATCH 04/32] =?UTF-8?q?Add=20MagicPlan=20SQS=20trigger=20to=20Hu?= =?UTF-8?q?bSpot=20orchestrator=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/config.py | 1 + .../hubspot_trigger_orchestrator/__init__.py | 0 .../tests/__init__.py | 0 .../tests/test_orchestrator.py | 148 ++++++++++++++++++ etl/hubspot/scripts/scraper/main.py | 21 +++ 5 files changed, 170 insertions(+) create mode 100644 backend/hubspot_trigger_orchestrator/__init__.py create mode 100644 backend/hubspot_trigger_orchestrator/tests/__init__.py create mode 100644 backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py diff --git a/backend/app/config.py b/backend/app/config.py index e939d6e4..21f12902 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -39,6 +39,7 @@ class Settings(BaseSettings): ENGINE_SQS_URL: str = "changeme" CATEGORISATION_SQS_URL: str = "changeme" PASHUB_TO_ARA_SQS_URL: str = "changeme" + MAGICPLAN_SQS_URL: str = "changeme" POSTCODE_SPLITTER_SQS_URL: str = "changeme" COMBINER_SQS_URL: str = "changeme" diff --git a/backend/hubspot_trigger_orchestrator/__init__.py b/backend/hubspot_trigger_orchestrator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/hubspot_trigger_orchestrator/tests/__init__.py b/backend/hubspot_trigger_orchestrator/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py b/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py new file mode 100644 index 00000000..6d18c4b4 --- /dev/null +++ b/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py @@ -0,0 +1,148 @@ +import json +import uuid +from typing import Any, Dict, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from etl.hubspot.scripts.scraper.main import handler + +COORDINATION_COMPLETE = "(v1) ioe/mtp complete" +DEAL_NAME = "123 Main Street" +UPRN = "12345678" +DEAL_ID = "999" +MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" + + +def make_hubspot_deal( + coordination_status: Optional[str] = None, **kwargs: Any +) -> Dict[str, Any]: + deal: Dict[str, Any] = { + "hs_object_id": DEAL_ID, + "dealname": DEAL_NAME, + "pashub_link": None, + **kwargs, + } + if coordination_status is not None: + deal["coordination_status__stage_1_"] = coordination_status + return deal + + +def make_db_deal(coordination_status: Optional[str] = None, **kwargs: Any) -> HubspotDealData: + return HubspotDealData( + id=uuid.uuid4(), + deal_id=DEAL_ID, + coordination_status=coordination_status, + **kwargs, + ) + + +def run_handler( + hubspot_deal: Dict[str, Any], + db_deal: Optional[HubspotDealData], + listing: Optional[dict], +) -> MagicMock: + mock_sqs = MagicMock() + mock_sqs.send_message.return_value = {"MessageId": "test-id"} + + with ( + patch("etl.hubspot.scripts.scraper.main.HubspotDataToDb") as mock_db_cls, + patch("etl.hubspot.scripts.scraper.main.HubspotClient") as mock_hs_cls, + patch("etl.hubspot.scripts.scraper.main.boto3") as mock_boto3, + patch("etl.hubspot.scripts.scraper.main.get_settings") as mock_settings, + ): + mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal + mock_db_cls.return_value.upsert_deal.return_value = None + mock_hs_cls.return_value.get_deal_and_company_and_listing.return_value = ( + hubspot_deal, + None, + listing, + ) + mock_boto3.client.return_value = mock_sqs + mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL + mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = "https://sqs.test/pashub" + + handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") + + return mock_sqs + + +# ======================= +# NEW DEAL PATH +# ======================= + + +def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + listing = {"national_uprn": UPRN} + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + ) + + +def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status="in progress") + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() + + +def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": None}), + ) + + +# ======================= +# EXISTING DEAL PATH +# ======================= + + +def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() -> None: + # Arrange + db_deal = make_db_deal(coordination_status="in progress") + hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + listing = {"national_uprn": UPRN} + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=MAGICPLAN_QUEUE_URL, + MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + ) + + +def test_existing_deal_already_in_coordination_complete_unrelated_change__no_sqs_message() -> None: + # Arrange + db_deal = make_db_deal(coordination_status=COORDINATION_COMPLETE, dealname="Old Name") + hubspot_deal = make_hubspot_deal( + coordination_status=COORDINATION_COMPLETE, dealname="New Name" + ) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 3ed208a2..cd76e26f 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -56,6 +56,13 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) _trigger_pashub_fetcher(sqs_client, hubspot_deal_id, hubspot_deal) + + coordination_status = (hubspot_deal.get("coordination_status__stage_1_") or "").lower() + if coordination_status in HubspotDealDiffer.COORDINATION_COMPLETE: + logger.info( + f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) else: # Deal already in db, check whether anything has changed logger.info( @@ -97,9 +104,23 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) + if HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=hubspot_deal, old_deal=db_deal + ): + logger.info( + f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + print("done") +def _trigger_magicplan_fetcher( + sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] +) -> None: + raise NotImplementedError + + def _trigger_pashub_fetcher( sqs_client: Any, deal_id: str, hubspot_deal: Dict[str, str] ) -> None: From a1a445f6f270f62b148e7d3f79eace348213e893 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:06:43 +0000 Subject: [PATCH 05/32] =?UTF-8?q?Add=20MagicPlan=20SQS=20trigger=20to=20Hu?= =?UTF-8?q?bSpot=20orchestrator=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/scripts/scraper/main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index cd76e26f..a39e8b37 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -118,7 +118,17 @@ def handler(body: dict[str, Any], context: Any) -> None: def _trigger_magicplan_fetcher( sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] ) -> None: - raise NotImplementedError + message_body = { + "address": hubspot_deal.get("dealname"), + "uprn": listing.get("national_uprn") if listing else None, + } + response = sqs_client.send_message( + QueueUrl=get_settings().MAGICPLAN_SQS_URL, + MessageBody=json.dumps(message_body), + ) + logger.info( + f"Sent message to MagicPlan queue. MessageId: {response['MessageId']}" + ) def _trigger_pashub_fetcher( From ed68a10127b132d883b927655c5f0ce2cdc41815 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:07:36 +0000 Subject: [PATCH 06/32] magic plan client terraform --- .../terraform/lambda/magic_plan/main.tf | 41 +++++++++++++++++++ .../terraform/lambda/magic_plan/outputs.tf | 9 ++++ .../terraform/lambda/magic_plan/provider.tf | 16 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 infrastructure/terraform/lambda/magic_plan/main.tf create mode 100644 infrastructure/terraform/lambda/magic_plan/outputs.tf create mode 100644 infrastructure/terraform/lambda/magic_plan/provider.tf diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf new file mode 100644 index 00000000..56adac1b --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -0,0 +1,41 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "magic_plan" + stage = var.stage + + image_uri = local.image_uri + + maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + MAGICPLAN_CUSTOMER_ID = var.magicplan_customer_id + MAGICPLAN_API_KEY = var.magicplan_api_key + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + } +} diff --git a/infrastructure/terraform/lambda/magic_plan/outputs.tf b/infrastructure/terraform/lambda/magic_plan/outputs.tf new file mode 100644 index 00000000..2082933f --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/outputs.tf @@ -0,0 +1,9 @@ +output "magic_plan_queue_url" { + value = module.lambda.queue_url + description = "URL of the MagicPlan SQS queue" +} + +output "magic_plan_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the MagicPlan SQS queue" +} diff --git a/infrastructure/terraform/lambda/magic_plan/provider.tf b/infrastructure/terraform/lambda/magic_plan/provider.tf new file mode 100644 index 00000000..9e7020ac --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "magic-plan-hubspot-trigger-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} From fd77fa51fdad5f1ea5dd3a6859f3051dd0665d84 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:07:53 +0000 Subject: [PATCH 07/32] magic plan client terraform --- .../terraform/lambda/magic_plan/variables.tf | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 infrastructure/terraform/lambda/magic_plan/variables.tf diff --git a/infrastructure/terraform/lambda/magic_plan/variables.tf b/infrastructure/terraform/lambda/magic_plan/variables.tf new file mode 100644 index 00000000..03f88e75 --- /dev/null +++ b/infrastructure/terraform/lambda/magic_plan/variables.tf @@ -0,0 +1,68 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null + description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} + +variable "magicplan_customer_id" { + type = string + sensitive = true +} + +variable "magicplan_api_key" { + type = string + sensitive = true +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} From feaa1ea68093f74087952597be691ee5a387fc8a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:12:13 +0000 Subject: [PATCH 08/32] Add MagicPlan Lambda Dockerfile, CI/CD jobs, and SQS IAM wiring in hubspot_deal_etl --- .github/workflows/deploy_terraform.yml | 40 ++++++++++++++++++- backend/magic_plan/handler/Dockerfile | 26 ++++++++++++ backend/magic_plan/handler/requirements.txt | 7 ++++ .../terraform/lambda/hubspot_deal_etl/main.tf | 26 +++++++++++- 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 backend/magic_plan/handler/Dockerfile create mode 100644 backend/magic_plan/handler/requirements.txt diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 398232c6..e0343974 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -537,11 +537,49 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # Build MagicPlan Lambda image + # ============================================================ + magic_plan_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/magic_plan/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy MagicPlan Lambda + # ============================================================ + magic_plan_lambda: + needs: [magic_plan_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: magic_plan + lambda_path: infrastructure/terraform/lambda/magic_plan + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.magic_plan_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }} + TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }} + # ============================================================ # Deploy Hubspot ETL Lambda # ============================================================ hubspot_etl_lambda: - needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda] + needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda, magic_plan_lambda] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: hubspot-etl-to-ara diff --git a/backend/magic_plan/handler/Dockerfile b/backend/magic_plan/handler/Dockerfile new file mode 100644 index 00000000..7c83ebe6 --- /dev/null +++ b/backend/magic_plan/handler/Dockerfile @@ -0,0 +1,26 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy + +# Install AWS Lambda RIE +ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie +RUN chmod +x /usr/local/bin/aws-lambda-rie + +# Set working directory (Lambda task root) +WORKDIR /var/task + +COPY backend/magic_plan/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utils/ utils/ +COPY backend/ backend/ +COPY datatypes/ datatypes/ + +# Local lambda entrypoint +# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] + +# AWS lambda entrypoint +ENTRYPOINT ["python", "-m", "awslambdaric"] + +# ----------------------------- +# Lambda handler +# ----------------------------- +CMD ["backend.magic_plan.handler.handler"] diff --git a/backend/magic_plan/handler/requirements.txt b/backend/magic_plan/handler/requirements.txt new file mode 100644 index 00000000..cfacf455 --- /dev/null +++ b/backend/magic_plan/handler/requirements.txt @@ -0,0 +1,7 @@ +awslambdaric +requests +sqlalchemy==2.0.36 +sqlmodel +psycopg2-binary==2.9.10 +pydantic-settings==2.6.0 +boto3==1.35.44 diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf index 48dd6b78..800dc3b6 100644 --- a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf @@ -12,7 +12,16 @@ data "terraform_remote_state" "pashub_to_ara" { config = { bucket = "pashub-to-ara-terraform-state" key = "env:/${var.stage}/terraform.tfstate" - region = "eu-west-2" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "magic_plan" { + backend = "s3" + config = { + bucket = "magic-plan-hubspot-trigger-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" } } @@ -49,6 +58,7 @@ module "hubspot_deal_etl" { HUBSPOT_API_KEY = var.hubspot_api_key PASHUB_TO_ARA_SQS_URL = data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_url + MAGICPLAN_SQS_URL = data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_url } } @@ -76,4 +86,18 @@ module "hubspot_deal_etl_sqs_policy" { resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_sqs_send" { role = module.hubspot_deal_etl.role_name policy_arn = module.hubspot_deal_etl_sqs_policy.policy_arn +} + +module "hubspot_deal_etl_magicplan_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "hubspot-deal-etl-magicplan-sqs-send-${var.stage}" + policy_description = "Allow HubSpot ETL Lambda to send messages to MagicPlan queue" + actions = ["sqs:SendMessage"] + resources = [data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_arn] +} + +resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_magicplan_sqs_send" { + role = module.hubspot_deal_etl.role_name + policy_arn = module.hubspot_deal_etl_magicplan_sqs_policy.policy_arn } \ No newline at end of file From e30e06cb6e61f24668bba039e0fb17c36f4a9307 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:24:04 +0000 Subject: [PATCH 09/32] simplify dockerfile as playwright not used --- backend/magic_plan/handler/Dockerfile | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/backend/magic_plan/handler/Dockerfile b/backend/magic_plan/handler/Dockerfile index 7c83ebe6..ffd85c02 100644 --- a/backend/magic_plan/handler/Dockerfile +++ b/backend/magic_plan/handler/Dockerfile @@ -1,10 +1,5 @@ -FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy +FROM public.ecr.aws/lambda/python:3.11 -# Install AWS Lambda RIE -ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie -RUN chmod +x /usr/local/bin/aws-lambda-rie - -# Set working directory (Lambda task root) WORKDIR /var/task COPY backend/magic_plan/handler/requirements.txt . @@ -14,13 +9,4 @@ COPY utils/ utils/ COPY backend/ backend/ COPY datatypes/ datatypes/ -# Local lambda entrypoint -# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] - -# AWS lambda entrypoint -ENTRYPOINT ["python", "-m", "awslambdaric"] - -# ----------------------------- -# Lambda handler -# ----------------------------- CMD ["backend.magic_plan.handler.handler"] From 74b3a7f297b90535f32208189c6d9a87c24f806a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:42:57 +0000 Subject: [PATCH 10/32] =?UTF-8?q?Add=20hubspot=5Fdeal=5Fid=20required=20fi?= =?UTF-8?q?eld=20to=20MagicPlanTriggerRequest=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../magic_plan/tests/test_magic_plan_trigger_request.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/magic_plan/tests/test_magic_plan_trigger_request.py b/backend/magic_plan/tests/test_magic_plan_trigger_request.py index 46a20a37..131ea93b 100644 --- a/backend/magic_plan/tests/test_magic_plan_trigger_request.py +++ b/backend/magic_plan/tests/test_magic_plan_trigger_request.py @@ -38,3 +38,11 @@ def test_extra_fields_ignored() -> None: req = MagicPlanTriggerRequest.model_validate(payload) # Assert assert req.address == "123 High St London SW1A 1AA" + + +def test_missing_hubspot_deal_id_raises() -> None: + # Arrange + payload = {"address": "123 High St London SW1A 1AA"} + # Act / Assert + with pytest.raises(ValidationError): + MagicPlanTriggerRequest.model_validate(payload) From 4a9cabe1979a1f0e12ef97f13dc634d4928b4fef Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 13:45:10 +0000 Subject: [PATCH 11/32] =?UTF-8?q?Add=20hubspot=5Fdeal=5Fid=20required=20fi?= =?UTF-8?q?eld=20to=20MagicPlanTriggerRequest=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- backend/magic_plan/handler.py | 2 +- backend/magic_plan/magic_plan_trigger_request.py | 1 + .../magic_plan/tests/test_magic_plan_trigger_request.py | 9 +++++---- etl/hubspot/scripts/scraper/main.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index a592cc6a..22933e13 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -28,7 +28,7 @@ if __name__ == "__main__": event = { "Records": [ { - "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ"}', + "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ", "hubspot_deal_id": "local-test-deal"}', "messageId": "local-test", } ] diff --git a/backend/magic_plan/magic_plan_trigger_request.py b/backend/magic_plan/magic_plan_trigger_request.py index bb0151e4..e93c055c 100644 --- a/backend/magic_plan/magic_plan_trigger_request.py +++ b/backend/magic_plan/magic_plan_trigger_request.py @@ -7,4 +7,5 @@ class MagicPlanTriggerRequest(BaseModel): model_config = ConfigDict(extra="ignore") address: str + hubspot_deal_id: str uprn: Optional[str] = None diff --git a/backend/magic_plan/tests/test_magic_plan_trigger_request.py b/backend/magic_plan/tests/test_magic_plan_trigger_request.py index 131ea93b..9fb2754a 100644 --- a/backend/magic_plan/tests/test_magic_plan_trigger_request.py +++ b/backend/magic_plan/tests/test_magic_plan_trigger_request.py @@ -6,17 +6,18 @@ from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerReques def test_valid_payload_with_address_only() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert assert req.address == "123 High St London SW1A 1AA" + assert req.hubspot_deal_id == "123456789" assert req.uprn is None def test_valid_payload_with_uprn() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA", "uprn": "100023336956"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789", "uprn": "100023336956"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert @@ -25,7 +26,7 @@ def test_valid_payload_with_uprn() -> None: def test_missing_address_raises() -> None: # Arrange - payload = {"uprn": "100023336956"} + payload = {"hubspot_deal_id": "123456789", "uprn": "100023336956"} # Act / Assert with pytest.raises(ValidationError): MagicPlanTriggerRequest.model_validate(payload) @@ -33,7 +34,7 @@ def test_missing_address_raises() -> None: def test_extra_fields_ignored() -> None: # Arrange - payload = {"address": "123 High St London SW1A 1AA", "unknown_field": "whatever"} + payload = {"address": "123 High St London SW1A 1AA", "hubspot_deal_id": "123456789", "unknown_field": "whatever"} # Act req = MagicPlanTriggerRequest.model_validate(payload) # Assert diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index a39e8b37..32007cd4 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -62,7 +62,7 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) - _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing, hubspot_deal_id) else: # Deal already in db, check whether anything has changed logger.info( @@ -110,16 +110,17 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) - _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing) + _trigger_magicplan_fetcher(sqs_client, hubspot_deal, listing, hubspot_deal_id) print("done") def _trigger_magicplan_fetcher( - sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]] + sqs_client: Any, hubspot_deal: Dict[str, str], listing: Optional[dict[str, str]], hubspot_deal_id: str ) -> None: message_body = { "address": hubspot_deal.get("dealname"), + "hubspot_deal_id": hubspot_deal_id, "uprn": listing.get("national_uprn") if listing else None, } response = sqs_client.send_message( From c3aae8fd51373a9b38cbdb023275bbd12172519c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:08:56 +0000 Subject: [PATCH 12/32] =?UTF-8?q?Expose=20get=5Fplan=5Fraw=20method=20on?= =?UTF-8?q?=20MagicPlanClient=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_client.py | 3 ++ .../tests/test_magic_plan_client.py | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/backend/magic_plan/magic_plan_client.py b/backend/magic_plan/magic_plan_client.py index 60f70fb1..172190fd 100644 --- a/backend/magic_plan/magic_plan_client.py +++ b/backend/magic_plan/magic_plan_client.py @@ -22,3 +22,6 @@ class MagicPlanClient: ) r.raise_for_status() return MagicPlanPlan.model_validate(r.json()["data"]) + + def get_plan_raw(self, plan_id: str) -> bytes: + raise NotImplementedError diff --git a/backend/magic_plan/tests/test_magic_plan_client.py b/backend/magic_plan/tests/test_magic_plan_client.py index 1be1448f..c96b9cdf 100644 --- a/backend/magic_plan/tests/test_magic_plan_client.py +++ b/backend/magic_plan/tests/test_magic_plan_client.py @@ -172,3 +172,55 @@ def test_get_plan_propagates_http_error( # Act / Assert with pytest.raises(requests.HTTPError): client.get_plan("some-id") + + +# --- get_plan_raw --- + + +def test_get_plan_raw_returns_bytes( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b'{"data": "raw"}' + plan_id = "a7285ed1-878d-47eb-8aa6-85ef9e187516" + # Act + result = client.get_plan_raw(plan_id) + # Assert + assert isinstance(result, bytes) + + +def test_get_plan_raw_calls_correct_url( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b"{}" + plan_id = "a7285ed1-878d-47eb-8aa6-85ef9e187516" + # Act + client.get_plan_raw(plan_id) + # Assert + mock_session.get.assert_called_once_with( + f"{BASE_URL}/plans/{plan_id}", params={"key": API_KEY} + ) + + +def test_get_plan_raw_calls_raise_for_status( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.content = b"{}" + # Act + client.get_plan_raw("a7285ed1-878d-47eb-8aa6-85ef9e187516") + # Assert + mock_session.get.return_value.raise_for_status.assert_called_once() + + +def test_get_plan_raw_propagates_http_error( + client: MagicPlanClient, mock_session: MagicMock +) -> None: + # Arrange + mock_session.get.return_value.raise_for_status.side_effect = requests.HTTPError( + "500" + ) + # Act / Assert + with pytest.raises(requests.HTTPError): + client.get_plan_raw("some-id") From f6c17be70a3d85e0638cea58f3edcdaf530c0aee Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:09:33 +0000 Subject: [PATCH 13/32] =?UTF-8?q?Expose=20get=5Fplan=5Fraw=20method=20on?= =?UTF-8?q?=20MagicPlanClient=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/magic_plan/magic_plan_client.py b/backend/magic_plan/magic_plan_client.py index 172190fd..06905e6a 100644 --- a/backend/magic_plan/magic_plan_client.py +++ b/backend/magic_plan/magic_plan_client.py @@ -17,11 +17,14 @@ class MagicPlanClient: return PlansListResponse.model_validate(r.json()["data"]) def get_plan(self, plan_id: str) -> MagicPlanPlan: + return MagicPlanPlan.model_validate(self._fetch_plan(plan_id).json()["data"]) + + def get_plan_raw(self, plan_id: str) -> bytes: + return self._fetch_plan(plan_id).content + + def _fetch_plan(self, plan_id: str) -> requests.Response: r = self._session.get( f"{_BASE_URL}/plans/{plan_id}", params={"key": self._api_key} ) r.raise_for_status() - return MagicPlanPlan.model_validate(r.json()["data"]) - - def get_plan_raw(self, plan_id: str) -> bytes: - raise NotImplementedError + return r From 7c9cb5b161b4e910d95c96c6633362000c4b9e18 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:14:42 +0000 Subject: [PATCH 14/32] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/db/models/uploaded_file.py | 2 + backend/magic_plan/handler.py | 2 +- backend/magic_plan/magic_plan_service.py | 13 +++- .../tests/test_magic_plan_service.py | 66 +++++++++++++++++-- 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/backend/app/db/models/uploaded_file.py b/backend/app/db/models/uploaded_file.py index a516a1df..c629f574 100644 --- a/backend/app/db/models/uploaded_file.py +++ b/backend/app/db/models/uploaded_file.py @@ -17,6 +17,7 @@ class FileTypeEnum(enum.Enum): ECMK_SITE_NOTE = "ecmk_site_note" ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" ECMK_SURVEY_XML = "ecmk_survey_xml" + MAGIC_PLAN_JSON = "magic_plan_json" class FileSourceEnum(enum.Enum): @@ -24,6 +25,7 @@ class FileSourceEnum(enum.Enum): SHAREPOINT = "sharepoint" HUBSPOT = "hubspot" ECMK = "ecmk" + MAGIC_PLAN = "magic_plan" class UploadedFile(Base): diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index 22933e13..45de8554 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -19,7 +19,7 @@ def handler(body: dict[str, Any], context: Any) -> str: customer_id=settings.MAGICPLAN_CUSTOMER_ID, api_key=settings.MAGICPLAN_API_KEY, ) - plan: Plan = MagicPlanService(client).run(payload.address, payload.uprn) + plan: Plan = MagicPlanService(client, s3_bucket="retrofit-energy-assessments-dev").run(payload) logger.info("Saved MagicPlan plan uid=%s", plan.uid) return plan.uid diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 91b3cd13..6be6486c 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,3 +1,4 @@ +import gzip from typing import Optional from datatypes.magicplan.api.response import ( @@ -12,23 +13,29 @@ from backend.app.db.connection import db_session from backend.app.db.functions.magic_plan_functions import save_plan from backend.magic_plan.address_matcher import find_matching_plan from backend.magic_plan.magic_plan_client import MagicPlanClient +from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest from utils.logger import setup_logger +from utils.s3 import save_data_to_s3 logger = setup_logger() class MagicPlanService: - def __init__(self, client: MagicPlanClient) -> None: + def __init__(self, client: MagicPlanClient, s3_bucket: str) -> None: self._client = client + self._s3_bucket = s3_bucket + + def run(self, request: MagicPlanTriggerRequest) -> Plan: + address = request.address + uprn = request.uprn - def run(self, address: str, uprn: Optional[str] = None) -> Plan: if uprn is not None: logger.info("MagicPlanService.run uprn=%s", uprn) plans_response: PlansListResponse = self._client.get_plans() matched: Optional[PlanSummary] = find_matching_plan( plans_response.plans, address - ) # TODO: use address2UPRN instead? or create AddressMatch domain class + ) if matched is None: raise ValueError(f"No MagicPlan found for address: {address!r}") diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 8e433b87..87e20506 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch import pytest @@ -10,9 +10,11 @@ from datatypes.magicplan.domain.models import Plan from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_service import MagicPlanService +from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest FIXTURE_DIR = Path(__file__).parents[2] / "magic_plan" PLAN_ID = "a7285ed1-878d-47eb-8aa6-85ef9e187516" +S3_BUCKET = "test-bucket" @pytest.fixture(scope="module") @@ -45,7 +47,17 @@ def mock_client() -> MagicMock: def _make_service(mock_client: MagicMock) -> MagicPlanService: - return MagicPlanService(client=mock_client) + return MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + + +def _make_request( + address: str = "2 Laburnum Way Bromley BR2 8BZ", + hubspot_deal_id: str = "deal-123", + uprn: str | None = None, +) -> MagicPlanTriggerRequest: + return MagicPlanTriggerRequest( + address=address, hubspot_deal_id=hubspot_deal_id, uprn=uprn + ) # --- no match --- @@ -57,7 +69,7 @@ def test_run_raises_when_no_plan_found(mock_client: MagicMock) -> None: service = _make_service(mock_client) # Act / Assert with pytest.raises(ValueError, match="No MagicPlan found"): - service.run("99 Nowhere Road London SW1A 1AA") + service.run(_make_request(address="99 Nowhere Road London SW1A 1AA")) # --- match found --- @@ -78,8 +90,10 @@ def test_run_fetches_plan_with_matched_id( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ") + service.run(_make_request()) # Assert mock_client.get_plan.assert_called_once_with(plan_summary.id) @@ -99,8 +113,10 @@ def test_run_returns_mapped_plan( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - result = service.run("2 Laburnum Way Bromley BR2 8BZ") + result = service.run(_make_request()) # Assert assert isinstance(result, Plan) assert result.uid == PLAN_ID @@ -120,8 +136,10 @@ def test_run_calls_save_plan_with_mapped_plan( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan") as mock_save, patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ") + service.run(_make_request()) # Assert — save_plan called with a Plan whose uid matches call_args = mock_save.call_args saved_plan: Plan = call_args[0][1] @@ -142,5 +160,39 @@ def test_run_accepts_uprn_without_error( return_value=plan_summary, ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" ): - service.run("2 Laburnum Way Bromley BR2 8BZ", uprn="100023336956") + service.run(_make_request(uprn="100023336956")) + + +# --- S3 upload --- + + +def test_run_uploads_to_s3_with_uprn_key( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + mock_client.get_plan_raw.return_value = b'{"raw": "data"}' + request = _make_request(uprn="100023336956") + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ) as mock_s3: + # Act + service.run(request) + # Assert + mock_s3.assert_called_once_with( + ANY, + S3_BUCKET, + f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz", + ) From 14a064fdefd1d6a9362cf3a4c078bbfc6da5b316 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:16:41 +0000 Subject: [PATCH 15/32] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 8 ++++++++ backend/magic_plan/tests/test_magic_plan_service.py | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 6be6486c..bb68fa42 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -41,8 +41,16 @@ class MagicPlanService: raise ValueError(f"No MagicPlan found for address: {address!r}") magic_plan: MagicPlanPlan = self._client.get_plan(matched.id) + raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) + compressed = gzip.compress(raw_bytes) + if uprn is not None: + s3_key = f"documents/uprn/{uprn}/magic_plan_{matched.id}.json.gz" + else: + s3_key = f"documents/hubspot_deal_id/{request.hubspot_deal_id}/magic_plan_{matched.id}.json.gz" + save_data_to_s3(compressed, self._s3_bucket, s3_key) + with db_session() as session: save_plan(session, plan) diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 87e20506..65c19b7a 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -43,7 +43,9 @@ def plan_summary() -> PlanSummary: @pytest.fixture() def mock_client() -> MagicMock: - return MagicMock(spec=MagicPlanClient) + client = MagicMock(spec=MagicPlanClient) + client.get_plan_raw.return_value = b"{}" + return client def _make_service(mock_client: MagicMock) -> MagicPlanService: From 03e8750c1ac233b408820eb56b4976b7b5a97d0b Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:17:51 +0000 Subject: [PATCH 16/32] =?UTF-8?q?Upload=20MagicPlan=20JSON=20to=20S3=20usi?= =?UTF-8?q?ng=20hubspot=5Fdeal=5Fid=20key=20when=20UPRN=20absent=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_magic_plan_service.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 65c19b7a..70099e91 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -198,3 +198,31 @@ def test_run_uploads_to_s3_with_uprn_key( S3_BUCKET, f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz", ) + + +def test_run_uploads_to_s3_with_deal_id_key_when_uprn_absent( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + request = _make_request(hubspot_deal_id="deal-456", uprn=None) + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ), patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ) as mock_s3: + # Act + service.run(request) + # Assert + mock_s3.assert_called_once_with( + ANY, + S3_BUCKET, + f"documents/hubspot_deal_id/deal-456/magic_plan_{plan_summary.id}.json.gz", + ) From 8ac77ce8b967eb87884443ad4e40e5e2d8c6fa29 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:19:49 +0000 Subject: [PATCH 17/32] =?UTF-8?q?Persist=20UploadedFile=20record=20for=20e?= =?UTF-8?q?ach=20MagicPlan=20S3=20upload=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 5 +++ .../tests/test_magic_plan_service.py | 45 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index bb68fa42..0aeed686 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -11,6 +11,11 @@ from datatypes.magicplan.domain.models import Plan from backend.app.db.connection import db_session from backend.app.db.functions.magic_plan_functions import save_plan +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) from backend.magic_plan.address_matcher import find_matching_plan from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index 70099e91..b7580546 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -8,6 +8,11 @@ from datatypes.magicplan.api.response import MagicPlanPlan, PlanSummary from datatypes.magicplan.domain.mapper import map_plan from datatypes.magicplan.domain.models import Plan +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) from backend.magic_plan.magic_plan_client import MagicPlanClient from backend.magic_plan.magic_plan_service import MagicPlanService from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest @@ -226,3 +231,43 @@ def test_run_uploads_to_s3_with_deal_id_key_when_uprn_absent( S3_BUCKET, f"documents/hubspot_deal_id/deal-456/magic_plan_{plan_summary.id}.json.gz", ) + + +# --- UploadedFile record --- + + +def test_run_creates_uploaded_file_record( + mock_client: MagicMock, + api_magic_plan: MagicPlanPlan, + plan_summary: PlanSummary, +) -> None: + # Arrange + mock_client.get_plans.return_value.plans = [plan_summary] + mock_client.get_plan.return_value = api_magic_plan + request = _make_request(hubspot_deal_id="deal-789", uprn="100023336956") + service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) + mock_session = MagicMock() + with patch( + "backend.magic_plan.magic_plan_service.find_matching_plan", + return_value=plan_summary, + ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch( + "backend.magic_plan.magic_plan_service.db_session" + ) as mock_db, patch( + "backend.magic_plan.magic_plan_service.save_data_to_s3" + ): + mock_db.return_value.__enter__.return_value = mock_session + # Act + service.run(request) + # Assert + added_objects = [call.args[0] for call in mock_session.add.call_args_list] + uploaded_file = next( + (obj for obj in added_objects if isinstance(obj, UploadedFile)), None + ) + assert uploaded_file is not None + assert uploaded_file.file_source == FileSourceEnum.MAGIC_PLAN.value + assert uploaded_file.file_type == FileTypeEnum.MAGIC_PLAN_JSON.value + assert uploaded_file.s3_file_bucket == S3_BUCKET + assert uploaded_file.s3_file_key == f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz" + assert uploaded_file.s3_upload_timestamp is not None + assert uploaded_file.uprn == 100023336956 + assert uploaded_file.hubspot_deal_id == "deal-789" From 337474e7733110e633601c5a7eaed26d6bbe4241 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:20:49 +0000 Subject: [PATCH 18/32] =?UTF-8?q?Persist=20UploadedFile=20record=20for=20e?= =?UTF-8?q?ach=20MagicPlan=20S3=20upload=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 0aeed686..7860fec9 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,4 +1,5 @@ import gzip +from datetime import datetime, timezone from typing import Optional from datatypes.magicplan.api.response import ( @@ -58,5 +59,16 @@ class MagicPlanService: with db_session() as session: save_plan(session, plan) + session.add( + UploadedFile( + s3_file_bucket=self._s3_bucket, + s3_file_key=s3_key, + s3_upload_timestamp=datetime.now(timezone.utc), + uprn=int(uprn) if uprn is not None else None, + hubspot_deal_id=request.hubspot_deal_id, + file_source=FileSourceEnum.MAGIC_PLAN.value, + file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, + ) + ) return plan From e1972e4349d86db57eca6ad5230bcc900a10935a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:23:32 +0000 Subject: [PATCH 19/32] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 47 +++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 7860fec9..9ac17e55 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -50,25 +50,38 @@ class MagicPlanService: raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) - compressed = gzip.compress(raw_bytes) - if uprn is not None: - s3_key = f"documents/uprn/{uprn}/magic_plan_{matched.id}.json.gz" - else: - s3_key = f"documents/hubspot_deal_id/{request.hubspot_deal_id}/magic_plan_{matched.id}.json.gz" - save_data_to_s3(compressed, self._s3_bucket, s3_key) + uploaded_file = self._upload_raw_plan_json( + plan_id=matched.id, + raw_bytes=raw_bytes, + uprn=uprn, + hubspot_deal_id=request.hubspot_deal_id, + ) with db_session() as session: save_plan(session, plan) - session.add( - UploadedFile( - s3_file_bucket=self._s3_bucket, - s3_file_key=s3_key, - s3_upload_timestamp=datetime.now(timezone.utc), - uprn=int(uprn) if uprn is not None else None, - hubspot_deal_id=request.hubspot_deal_id, - file_source=FileSourceEnum.MAGIC_PLAN.value, - file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, - ) - ) + session.add(uploaded_file) return plan + + def _upload_raw_plan_json( + self, + plan_id: str, + raw_bytes: bytes, + uprn: Optional[str], + hubspot_deal_id: str, + ) -> UploadedFile: + compressed = gzip.compress(raw_bytes) + if uprn is not None: + s3_key = f"documents/uprn/{uprn}/magic_plan_{plan_id}.json.gz" + else: + s3_key = f"documents/hubspot_deal_id/{hubspot_deal_id}/magic_plan_{plan_id}.json.gz" + save_data_to_s3(compressed, self._s3_bucket, s3_key) + return UploadedFile( + s3_file_bucket=self._s3_bucket, + s3_file_key=s3_key, + s3_upload_timestamp=datetime.now(timezone.utc), + uprn=int(uprn) if uprn is not None else None, + hubspot_deal_id=hubspot_deal_id, + file_source=FileSourceEnum.MAGIC_PLAN.value, + file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, + ) From 9f62e3c31abe61a9f67faf38e3bbc77730a1b64d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 14:30:59 +0000 Subject: [PATCH 20/32] typehint --- backend/magic_plan/magic_plan_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 9ac17e55..6ed25c0c 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -50,7 +50,7 @@ class MagicPlanService: raw_bytes: bytes = self._client.get_plan_raw(matched.id) plan: Plan = map_plan(magic_plan) - uploaded_file = self._upload_raw_plan_json( + uploaded_file: UploadedFile = self._upload_raw_plan_json( plan_id=matched.id, raw_bytes=raw_bytes, uprn=uprn, From ce2b61d60b10b5608f390a267ad2fe2d92b7d164 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:07:09 +0000 Subject: [PATCH 21/32] =?UTF-8?q?Upload=20gzip-compressed=20MagicPlan=20JS?= =?UTF-8?q?ON=20to=20S3=20-=20only=20make=20one=20API=20call=20?= =?UTF-8?q?=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/magic_plan/magic_plan_service.py | 5 ++++- backend/magic_plan/tests/test_magic_plan_service.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py index 6ed25c0c..fb0a7610 100644 --- a/backend/magic_plan/magic_plan_service.py +++ b/backend/magic_plan/magic_plan_service.py @@ -1,4 +1,5 @@ import gzip +import json from datetime import datetime, timezone from typing import Optional @@ -46,8 +47,10 @@ class MagicPlanService: if matched is None: raise ValueError(f"No MagicPlan found for address: {address!r}") - magic_plan: MagicPlanPlan = self._client.get_plan(matched.id) raw_bytes: bytes = self._client.get_plan_raw(matched.id) + magic_plan: MagicPlanPlan = MagicPlanPlan.model_validate( + json.loads(raw_bytes)["data"] + ) plan: Plan = map_plan(magic_plan) uploaded_file: UploadedFile = self._upload_raw_plan_json( diff --git a/backend/magic_plan/tests/test_magic_plan_service.py b/backend/magic_plan/tests/test_magic_plan_service.py index b7580546..f6954824 100644 --- a/backend/magic_plan/tests/test_magic_plan_service.py +++ b/backend/magic_plan/tests/test_magic_plan_service.py @@ -49,7 +49,9 @@ def plan_summary() -> PlanSummary: @pytest.fixture() def mock_client() -> MagicMock: client = MagicMock(spec=MagicPlanClient) - client.get_plan_raw.return_value = b"{}" + client.get_plan_raw.return_value = ( + FIXTURE_DIR / "magicplan_api_plan_response_example.json" + ).read_bytes() return client @@ -102,7 +104,7 @@ def test_run_fetches_plan_with_matched_id( ): service.run(_make_request()) # Assert - mock_client.get_plan.assert_called_once_with(plan_summary.id) + mock_client.get_plan_raw.assert_called_once_with(plan_summary.id) def test_run_returns_mapped_plan( @@ -183,8 +185,6 @@ def test_run_uploads_to_s3_with_uprn_key( ) -> None: # Arrange mock_client.get_plans.return_value.plans = [plan_summary] - mock_client.get_plan.return_value = api_magic_plan - mock_client.get_plan_raw.return_value = b'{"raw": "data"}' request = _make_request(uprn="100023336956") service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET) with patch( From 1243690d100a0d73b6a91100e02ed289f160b87a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:41:12 +0000 Subject: [PATCH 22/32] give handler permission to write to s3 bucket --- backend/magic_plan/handler.py | 1 + infrastructure/terraform/lambda/magic_plan/main.tf | 5 +++++ infrastructure/terraform/shared/main.tf | 14 ++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py index 45de8554..f2c03b90 100644 --- a/backend/magic_plan/handler.py +++ b/backend/magic_plan/handler.py @@ -19,6 +19,7 @@ def handler(body: dict[str, Any], context: Any) -> str: customer_id=settings.MAGICPLAN_CUSTOMER_ID, api_key=settings.MAGICPLAN_API_KEY, ) + # TODO: read s3_bucket from env var so staging/prod use the correct bucket plan: Plan = MagicPlanService(client, s3_bucket="retrofit-energy-assessments-dev").run(payload) logger.info("Saved MagicPlan plan uid=%s", plan.uid) return plan.uid diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf index 56adac1b..e2017b42 100644 --- a/infrastructure/terraform/lambda/magic_plan/main.tf +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -15,6 +15,11 @@ locals { db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) } +resource "aws_iam_role_policy_attachment" "magic_plan_s3_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.magic_plan_s3_write_arn +} + module "lambda" { source = "../../modules/lambda_with_sqs" diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 050ebdc2..e32ce395 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -745,4 +745,18 @@ module "magic_plan_client_registry" { source = "../modules/container_registry" name = "magic-plan" stage = var.stage +} + +module "magic_plan_s3_write" { + source = "../modules/s3_iam_policy" + + policy_name = "MagicPlanWriteS3" + policy_description = "Allow MagicPlan Lambda to write to retrofit energy assessments bucket" + bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] + actions = ["s3:PutObject", "s3:AbortMultipartUpload"] + resource_paths = ["/*"] +} + +output "magic_plan_s3_write_arn" { + value = module.magic_plan_s3_write.policy_arn } \ No newline at end of file From aadf73ed87db510efc17a1579122d7bc5e65ca15 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 8 May 2026 15:44:14 +0000 Subject: [PATCH 23/32] combine s3 write policies into one and apply to pashub and magicplan lambdas --- .../terraform/lambda/magic_plan/main.tf | 2 +- .../terraform/lambda/pashub_to_ara/main.tf | 2 +- infrastructure/terraform/shared/main.tf | 23 ++++--------------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/infrastructure/terraform/lambda/magic_plan/main.tf index e2017b42..48fc3867 100644 --- a/infrastructure/terraform/lambda/magic_plan/main.tf +++ b/infrastructure/terraform/lambda/magic_plan/main.tf @@ -17,7 +17,7 @@ locals { resource "aws_iam_role_policy_attachment" "magic_plan_s3_write" { role = module.lambda.role_name - policy_arn = data.terraform_remote_state.shared.outputs.magic_plan_s3_write_arn + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn } module "lambda" { diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/infrastructure/terraform/lambda/pashub_to_ara/main.tf index 1a457617..902d7845 100644 --- a/infrastructure/terraform/lambda/pashub_to_ara/main.tf +++ b/infrastructure/terraform/lambda/pashub_to_ara/main.tf @@ -54,5 +54,5 @@ module "lambda" { resource "aws_iam_role_policy_attachment" "pashub_to_ara_s3_write" { role = module.lambda.role_name - policy_arn = data.terraform_remote_state.shared.outputs.pashub_to_ara_s3_write_arn + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn } diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index e32ce395..2c3200de 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -568,18 +568,18 @@ module "pashub_to_ara_registry" { stage = var.stage } -module "pashub_to_ara_s3_write" { +module "energy_assessments_s3_write" { source = "../modules/s3_iam_policy" - policy_name = "PashubToAraWriteS3" - policy_description = "Allow PasHub to ARA Lambda to write to retrofit energy assessments bucket" + policy_name = "EnergyAssessmentsWriteS3" + policy_description = "Allow lambdas to write to retrofit energy assessments bucket" bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] actions = ["s3:PutObject", "s3:AbortMultipartUpload"] resource_paths = ["/*"] } -output "pashub_to_ara_s3_write_arn" { - value = module.pashub_to_ara_s3_write.policy_arn +output "energy_assessments_s3_write_arn" { + value = module.energy_assessments_s3_write.policy_arn } ################################################ @@ -747,16 +747,3 @@ module "magic_plan_client_registry" { stage = var.stage } -module "magic_plan_s3_write" { - source = "../modules/s3_iam_policy" - - policy_name = "MagicPlanWriteS3" - policy_description = "Allow MagicPlan Lambda to write to retrofit energy assessments bucket" - bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] - actions = ["s3:PutObject", "s3:AbortMultipartUpload"] - resource_paths = ["/*"] -} - -output "magic_plan_s3_write_arn" { - value = module.magic_plan_s3_write.policy_arn -} \ No newline at end of file From 2049553176ee88e1fefa2062f300d036e58206a1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 11 May 2026 09:25:41 +0000 Subject: [PATCH 24/32] =?UTF-8?q?Trigger=20MagicPlan=20on=20outcome=20"sur?= =?UTF-8?q?veyed"=20transition=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 7 +-- etl/hubspot/tests/test_hubspot_deal_differ.py | 49 ++++++++++--------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 5435a46d..ba3dc27a 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -198,12 +198,7 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - new_status = (new_deal.get("coordination_status__stage_1_") or "").lower() - old_status = (old_deal.coordination_status or "").lower() - return ( - new_status in HubspotDealDiffer.COORDINATION_COMPLETE - and old_status not in HubspotDealDiffer.COORDINATION_COMPLETE - ) + raise NotImplementedError @staticmethod def _lodgement_completed( diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 273a82a0..94952424 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -275,15 +275,12 @@ def test_pashub_trigger__coordination_design_lodgement_not_completed_and_pashub_ # ========================== -def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() -> None: +def test_magicplan_trigger__outcome_transitions_to_surveyed__returns_true() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal(id=deal_id, coordination_status="in progress") - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, - ) + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="surveyed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -295,20 +292,12 @@ def test_magicplan_trigger__transitions_to_coordination_complete__returns_true() assert result is True -def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__returns_false() -> None: +def test_magicplan_trigger__outcome_already_surveyed__returns_false() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal( - id=deal_id, - coordination_status="(v1) ioe/mtp complete", - outcome="pending", - ) - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "(v1) ioe/mtp complete"}, - outcome="won", - ) + old_deal = make_old_deal(id=deal_id, outcome="surveyed") + new_deal = make_new_deal(deal_id, outcome="surveyed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -320,15 +309,12 @@ def test_magicplan_trigger__already_in_coordination_complete_unrelated_change__r assert result is False -def test_magicplan_trigger__transitions_to_non_complete_coordination_status__returns_false() -> None: +def test_magicplan_trigger__outcome_transitions_to_non_surveyed__returns_false() -> None: deal_id = uuid.uuid4() # Arrange - old_deal = make_old_deal(id=deal_id, coordination_status="in progress") - new_deal = make_new_deal( - deal_id, - **{"coordination_status__stage_1_": "design submitted"}, - ) + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="assessed") # Act result = HubspotDealDiffer.check_for_magicplan_trigger( @@ -340,6 +326,23 @@ def test_magicplan_trigger__transitions_to_non_complete_coordination_status__ret assert result is False +def test_magicplan_trigger__outcome_surveyed_uppercase__returns_true() -> None: + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal(id=deal_id, outcome="assessed") + new_deal = make_new_deal(deal_id, outcome="SURVEYED") + + # Act + result = HubspotDealDiffer.check_for_magicplan_trigger( + new_deal=new_deal, + old_deal=old_deal, + ) + + # Assert + assert result is True + + # ======================= # DB UPDATE TRIGGER TESTS # ======================= From c15ffdf2c01765b0baa3e1fb371afe8c54e462c4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 11 May 2026 09:26:20 +0000 Subject: [PATCH 25/32] =?UTF-8?q?Trigger=20MagicPlan=20on=20outcome=20"sur?= =?UTF-8?q?veyed"=20transition=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index ba3dc27a..724a3e68 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -198,7 +198,9 @@ class HubspotDealDiffer: def check_for_magicplan_trigger( new_deal: Dict[str, str], old_deal: HubspotDealData ) -> bool: - raise NotImplementedError + new_outcome = (new_deal.get("outcome") or "").lower() + old_outcome = (old_deal.outcome or "").lower() + return new_outcome == "surveyed" and old_outcome != "surveyed" @staticmethod def _lodgement_completed( From 197e9a0e009565b72ad461f1f49ec26cb13226da Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 11 May 2026 15:21:16 +0000 Subject: [PATCH 26/32] added histroci_epc.csv --- datatypes/epc/schema/tests/fixtures/historic_epc.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 datatypes/epc/schema/tests/fixtures/historic_epc.csv diff --git a/datatypes/epc/schema/tests/fixtures/historic_epc.csv b/datatypes/epc/schema/tests/fixtures/historic_epc.csv new file mode 100644 index 00000000..b4c10739 --- /dev/null +++ b/datatypes/epc/schema/tests/fixtures/historic_epc.csv @@ -0,0 +1,2 @@ +LMK_KEY,ADDRESS1,ADDRESS2,ADDRESS3,POSTCODE,BUILDING_REFERENCE_NUMBER,CURRENT_ENERGY_RATING,POTENTIAL_ENERGY_RATING,CURRENT_ENERGY_EFFICIENCY,POTENTIAL_ENERGY_EFFICIENCY,PROPERTY_TYPE,BUILT_FORM,INSPECTION_DATE,LOCAL_AUTHORITY,CONSTITUENCY,COUNTY,LODGEMENT_DATE,TRANSACTION_TYPE,ENVIRONMENT_IMPACT_CURRENT,ENVIRONMENT_IMPACT_POTENTIAL,ENERGY_CONSUMPTION_CURRENT,ENERGY_CONSUMPTION_POTENTIAL,CO2_EMISSIONS_CURRENT,CO2_EMISS_CURR_PER_FLOOR_AREA,CO2_EMISSIONS_POTENTIAL,LIGHTING_COST_CURRENT,LIGHTING_COST_POTENTIAL,HEATING_COST_CURRENT,HEATING_COST_POTENTIAL,HOT_WATER_COST_CURRENT,HOT_WATER_COST_POTENTIAL,TOTAL_FLOOR_AREA,ENERGY_TARIFF,MAINS_GAS_FLAG,FLOOR_LEVEL,FLAT_TOP_STOREY,FLAT_STOREY_COUNT,MAIN_HEATING_CONTROLS,MULTI_GLAZE_PROPORTION,GLAZED_TYPE,GLAZED_AREA,EXTENSION_COUNT,NUMBER_HABITABLE_ROOMS,NUMBER_HEATED_ROOMS,LOW_ENERGY_LIGHTING,NUMBER_OPEN_FIREPLACES,HOTWATER_DESCRIPTION,HOT_WATER_ENERGY_EFF,HOT_WATER_ENV_EFF,FLOOR_DESCRIPTION,FLOOR_ENERGY_EFF,FLOOR_ENV_EFF,WINDOWS_DESCRIPTION,WINDOWS_ENERGY_EFF,WINDOWS_ENV_EFF,WALLS_DESCRIPTION,WALLS_ENERGY_EFF,WALLS_ENV_EFF,SECONDHEAT_DESCRIPTION,SHEATING_ENERGY_EFF,SHEATING_ENV_EFF,ROOF_DESCRIPTION,ROOF_ENERGY_EFF,ROOF_ENV_EFF,MAINHEAT_DESCRIPTION,MAINHEAT_ENERGY_EFF,MAINHEAT_ENV_EFF,MAINHEATCONT_DESCRIPTION,MAINHEATC_ENERGY_EFF,MAINHEATC_ENV_EFF,LIGHTING_DESCRIPTION,LIGHTING_ENERGY_EFF,LIGHTING_ENV_EFF,MAIN_FUEL,WIND_TURBINE_COUNT,HEAT_LOSS_CORRIDOR,UNHEATED_CORRIDOR_LENGTH,FLOOR_HEIGHT,PHOTO_SUPPLY,SOLAR_WATER_HEATING_FLAG,MECHANICAL_VENTILATION,ADDRESS,LOCAL_AUTHORITY_LABEL,CONSTITUENCY_LABEL,POSTTOWN,CONSTRUCTION_AGE_BAND,LODGEMENT_DATETIME,TENURE,FIXED_LIGHTING_OUTLETS_COUNT,LOW_ENERGY_FIXED_LIGHT_COUNT,UPRN,UPRN_SOURCE,REPORT_TYPE +9292c3bf26a8876ce59274401ea73e3de5bd0b3e52a507c2162a46e57db8ea2f,47 GORDON ROAD,ALFORD,,AB33 8AL,10001111325,E,B,42,87,House,Semi-Detached,2021-04-11,,Unknown,,2021-04-12,ECO assessment,49,69,450,299,5.5,76,3.6,69,77,1579,715,349,118,72.0,Single,N,,,,,100.0,"double glazing, unknown install date",Normal,0.0,3.0,3.0,86.0,0.0,"Electric immersion, standard tariff",Very Poor,Poor,"Solid, no insulation (assumed)",,,Fully double glazed,Average,Average,"Granite or whinstone, as built, partial insulation (assumed)",Average,Average,,,,"Pitched, 100 mm loft insulation",Average,Average,"Room heaters, electric",Very Poor,Poor,Appliance thermostats,Good,Good,Low energy lighting in 86% of fixed outlets,Very Good,Very Good,electricity (not community),0.0,,,2.4,0.0,N,natural,"47 GORDON ROAD, ALFORD",,,ALFORD,England and Wales: 1976-1982,2021-04-12 21:45:35,Rented (private),7.0,,151020766.0,Energy Assessor,100 \ No newline at end of file From f0300eb8ff4749da93a49a259c88f448ab7dee08 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:57:24 +0000 Subject: [PATCH 27/32] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/hubspot_deal_differ.py | 16 ++--- .../hubspot/tests/test_scraper_handler.py | 65 +++++++++---------- pytest.ini | 2 +- 3 files changed, 39 insertions(+), 44 deletions(-) rename backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py => etl/hubspot/tests/test_scraper_handler.py (61%) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 724a3e68..da0072c1 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -162,6 +162,14 @@ class HubspotDealDiffer: return False + @staticmethod + def check_for_magicplan_trigger( + new_deal: Dict[str, str], old_deal: HubspotDealData + ) -> bool: + new_outcome = (new_deal.get("outcome") or "").lower() + old_outcome = (old_deal.outcome or "").lower() + return new_outcome == "surveyed" and old_outcome != "surveyed" + @staticmethod def _has_valid_pashub_link(new_pashub_link: str) -> bool: return bool(new_pashub_link) @@ -194,14 +202,6 @@ class HubspotDealDiffer: and new_status != old_deal.design_status ) - @staticmethod - def check_for_magicplan_trigger( - new_deal: Dict[str, str], old_deal: HubspotDealData - ) -> bool: - new_outcome = (new_deal.get("outcome") or "").lower() - old_outcome = (old_deal.outcome or "").lower() - return new_outcome == "surveyed" and old_outcome != "surveyed" - @staticmethod def _lodgement_completed( new_deal: Dict[str, str], old_deal: HubspotDealData diff --git a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py b/etl/hubspot/tests/test_scraper_handler.py similarity index 61% rename from backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py rename to etl/hubspot/tests/test_scraper_handler.py index 6d18c4b4..e2f80d07 100644 --- a/backend/hubspot_trigger_orchestrator/tests/test_orchestrator.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -3,37 +3,28 @@ import uuid from typing import Any, Dict, Optional from unittest.mock import MagicMock, patch -import pytest - from backend.app.db.models.hubspot_deal_data import HubspotDealData from etl.hubspot.scripts.scraper.main import handler -COORDINATION_COMPLETE = "(v1) ioe/mtp complete" DEAL_NAME = "123 Main Street" UPRN = "12345678" DEAL_ID = "999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" -def make_hubspot_deal( - coordination_status: Optional[str] = None, **kwargs: Any -) -> Dict[str, Any]: - deal: Dict[str, Any] = { +def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: + return { "hs_object_id": DEAL_ID, "dealname": DEAL_NAME, "pashub_link": None, **kwargs, } - if coordination_status is not None: - deal["coordination_status__stage_1_"] = coordination_status - return deal -def make_db_deal(coordination_status: Optional[str] = None, **kwargs: Any) -> HubspotDealData: +def make_db_deal(**kwargs: Any) -> HubspotDealData: return HubspotDealData( id=uuid.uuid4(), deal_id=DEAL_ID, - coordination_status=coordination_status, **kwargs, ) @@ -68,14 +59,14 @@ def run_handler( return mock_sqs -# ======================= -# NEW DEAL PATH -# ======================= +# ==================================== +# NEW DEAL PATH - MagicPlan trigger +# ==================================== -def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: +def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} # Act @@ -84,13 +75,15 @@ def test_new_deal_in_coordination_complete__sends_sqs_message() -> None: # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": UPRN} + ), ) -def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: +def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status="in progress") + hubspot_deal = make_hubspot_deal(outcome="assessed") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) @@ -99,9 +92,9 @@ def test_new_deal_not_in_coordination_complete__no_sqs_message() -> None: mock_sqs.send_message.assert_not_called() -def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: +def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_null() -> None: # Arrange - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + hubspot_deal = make_hubspot_deal(outcome="surveyed") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) @@ -109,19 +102,21 @@ def test_new_deal_with_no_listing__uprn_is_none_in_message() -> None: # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": None}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": None} + ), ) -# ======================= -# EXISTING DEAL PATH -# ======================= +# ========================================== +# EXISTING DEAL PATH - MagicPlan trigger +# ========================================== -def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() -> None: +def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> None: # Arrange - db_deal = make_db_deal(coordination_status="in progress") - hubspot_deal = make_hubspot_deal(coordination_status=COORDINATION_COMPLETE) + db_deal = make_db_deal(outcome="assessed") + hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} # Act @@ -130,16 +125,16 @@ def test_existing_deal_transitions_to_coordination_complete__sends_sqs_message() # Assert mock_sqs.send_message.assert_called_once_with( QueueUrl=MAGICPLAN_QUEUE_URL, - MessageBody=json.dumps({"address": DEAL_NAME, "uprn": UPRN}), + MessageBody=json.dumps( + {"address": DEAL_NAME, "hubspot_deal_id": DEAL_ID, "uprn": UPRN} + ), ) -def test_existing_deal_already_in_coordination_complete_unrelated_change__no_sqs_message() -> None: +def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_trigger_magicplan() -> None: # Arrange - db_deal = make_db_deal(coordination_status=COORDINATION_COMPLETE, dealname="Old Name") - hubspot_deal = make_hubspot_deal( - coordination_status=COORDINATION_COMPLETE, dealname="New Name" - ) + db_deal = make_db_deal(outcome="surveyed", dealname="Old Name") + hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") # Act mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) diff --git a/pytest.ini b/pytest.ini index 398c5b71..e2a4a25d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,6 +3,6 @@ pythonpath = . log_cli = true log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial -testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests etl/hubspot/tests backend/hubspot_trigger_orchestrator/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests datatypes/epc/domain/tests backend/ecmk_fetcher/tests/ backend/pashub_fetcher/tests backend/documents_parser/tests backend/magic_plan/tests datatypes/magicplan/api/tests datatypes/magicplan/domain/tests backend/app/db/functions/tests +testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests etl/hubspot/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests datatypes/epc/domain/tests backend/ecmk_fetcher/tests/ backend/pashub_fetcher/tests backend/documents_parser/tests backend/magic_plan/tests datatypes/magicplan/api/tests datatypes/magicplan/domain/tests backend/app/db/functions/tests markers = integration: mark a test as an integration test From 9386846044e087b38a15bbfae3dd306f376cd205 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:58:15 +0000 Subject: [PATCH 28/32] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etl/hubspot/scripts/scraper/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 32007cd4..86844352 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -57,8 +57,7 @@ def handler(body: dict[str, Any], context: Any) -> None: ) _trigger_pashub_fetcher(sqs_client, hubspot_deal_id, hubspot_deal) - coordination_status = (hubspot_deal.get("coordination_status__stage_1_") or "").lower() - if coordination_status in HubspotDealDiffer.COORDINATION_COMPLETE: + if (hubspot_deal.get("outcome") or "").lower() == "surveyed": logger.info( f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" ) From 9501146ec815b2f2998017acb646571c922ba277 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 08:59:32 +0000 Subject: [PATCH 29/32] =?UTF-8?q?Replace=20new-deal=20MagicPlan=20trigger?= =?UTF-8?q?=20to=20use=20outcome=3D=3D"surveyed"=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/hubspot_trigger_orchestrator/__init__.py | 0 backend/hubspot_trigger_orchestrator/tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/hubspot_trigger_orchestrator/__init__.py delete mode 100644 backend/hubspot_trigger_orchestrator/tests/__init__.py diff --git a/backend/hubspot_trigger_orchestrator/__init__.py b/backend/hubspot_trigger_orchestrator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/hubspot_trigger_orchestrator/tests/__init__.py b/backend/hubspot_trigger_orchestrator/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 From 71aadfe78d1237cca0721707d97b7fc01e6bb6c3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 09:31:47 +0000 Subject: [PATCH 30/32] add pashub functions to orchestrator tests, and rename existing magicplan ones --- etl/hubspot/tests/test_scraper_handler.py | 96 +++++++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index e2f80d07..4810d171 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -9,7 +9,9 @@ from etl.hubspot.scripts.scraper.main import handler DEAL_NAME = "123 Main Street" UPRN = "12345678" DEAL_ID = "999" +PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" +PASHUB_QUEUE_URL = "https://sqs.test/pashub" def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: @@ -52,7 +54,7 @@ def run_handler( ) mock_boto3.client.return_value = mock_sqs mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL - mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = "https://sqs.test/pashub" + mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = PASHUB_QUEUE_URL handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") @@ -64,7 +66,7 @@ def run_handler( # ==================================== -def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: +def test_new_deal_surveyed__sends_magicplan_sqs() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="surveyed") listing = {"national_uprn": UPRN} @@ -81,7 +83,7 @@ def test_new_deal__outcome_is_surveyed__triggers_magicplan() -> None: ) -def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None: +def test_new_deal_not_surveyed__no_magicplan_sqs() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="assessed") @@ -92,7 +94,7 @@ def test_new_deal__outcome_is_not_surveyed__does_not_trigger_magicplan() -> None mock_sqs.send_message.assert_not_called() -def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_null() -> None: +def test_new_deal_surveyed_no_listing__magicplan_sqs_uprn_is_null() -> None: # Arrange hubspot_deal = make_hubspot_deal(outcome="surveyed") @@ -113,7 +115,7 @@ def test_new_deal__outcome_is_surveyed__no_listing__magicplan_message_uprn_is_nu # ========================================== -def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> None: +def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None: # Arrange db_deal = make_db_deal(outcome="assessed") hubspot_deal = make_hubspot_deal(outcome="surveyed") @@ -131,7 +133,7 @@ def test_existing_deal__outcome_transitions_to_surveyed__triggers_magicplan() -> ) -def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_trigger_magicplan() -> None: +def test_existing_deal_already_surveyed__no_magicplan_sqs() -> None: # Arrange db_deal = make_db_deal(outcome="surveyed", dealname="Old Name") hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") @@ -141,3 +143,85 @@ def test_existing_deal__outcome_already_surveyed__unrelated_change__does_not_tri # Assert mock_sqs.send_message.assert_not_called() + + +# ==================================== +# NEW DEAL PATH - PasHub trigger +# ==================================== + + +def test_new_deal_with_pashub_link__sends_pashub_sqs() -> None: + # Arrange + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=PASHUB_QUEUE_URL, + MessageBody=json.dumps( + { + "pashub_link": PASHUB_LINK, + "address": None, + "hubspot_deal_id": DEAL_ID, + "sharepoint_link": None, + "uprn": None, + "landlord_property_id": None, + "deal_stage": None, + } + ), + ) + + +def test_new_deal_no_pashub_link__no_pashub_sqs() -> None: + # Arrange + hubspot_deal = make_hubspot_deal() + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() + + +# ========================================== +# EXISTING DEAL PATH - PasHub trigger +# ========================================== + + +def test_existing_deal_pashub_link_added__sends_pashub_sqs() -> None: + # Arrange + db_deal = make_db_deal(pashub_link=None) + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=PASHUB_QUEUE_URL, + MessageBody=json.dumps( + { + "pashub_link": PASHUB_LINK, + "address": None, + "hubspot_deal_id": DEAL_ID, + "sharepoint_link": None, + "uprn": None, + "landlord_property_id": None, + "deal_stage": None, + } + ), + ) + + +def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: + # Arrange + db_deal = make_db_deal(pashub_link=PASHUB_LINK, dealname="Old Name") + hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name") + + # Act + mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + + # Assert + mock_sqs.send_message.assert_not_called() From f5bbd2efb3921fb8a419f478bf1a7e4aeb2e7c49 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 11:23:50 +0000 Subject: [PATCH 31/32] add missing tf_vars to deploy_lambda workflow --- .github/workflows/_deploy_lambda.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 3a407c5a..1cc7d462 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -82,6 +82,12 @@ on: required: false TF_VAR_hubspot_api_key: required: false + + TF_VAR_magicplan_customer_id: + required: false + + TF_VAR_magicplan_api_key: + required: false jobs: deploy: runs-on: ubuntu-latest @@ -149,6 +155,8 @@ jobs: TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} TF_VAR_hubspot_api_key: ${{ secrets.TF_VAR_hubspot_api_key }} + TF_VAR_magicplan_customer_id: ${{ secrets.TF_VAR_magicplan_customer_id }} + TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }} run: | ECR_REPO_URL_VAR="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then @@ -195,6 +203,8 @@ jobs: TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} TF_VAR_hubspot_api_key: ${{ secrets.TF_VAR_hubspot_api_key }} + TF_VAR_magicplan_customer_id: ${{ secrets.TF_VAR_magicplan_customer_id }} + TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }} run: | EXTRA_VARS="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then From ec7acabaf8215a022a5f1bc25c44bb298346a8c7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 12 May 2026 11:48:39 +0000 Subject: [PATCH 32/32] reinstate deleted policy so it can be unattached from entities --- infrastructure/terraform/shared/main.tf | 26 ++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 2c3200de..0a9e87f6 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -280,6 +280,21 @@ output "retrofit_energy_assessments_bucket_name" { description = "Name of the retrofit energy assessments bucket" } +module "energy_assessments_s3_write" { + source = "../modules/s3_iam_policy" + + policy_name = "EnergyAssessmentsWriteS3" + policy_description = "Allow lambdas to write to retrofit energy assessments bucket" + bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] + actions = ["s3:PutObject", "s3:AbortMultipartUpload"] + resource_paths = ["/*"] +} + +output "energy_assessments_s3_write_arn" { + value = module.energy_assessments_s3_write.policy_arn +} + + # Set up the route53 record for the API module "route53" { @@ -568,18 +583,19 @@ module "pashub_to_ara_registry" { stage = var.stage } -module "energy_assessments_s3_write" { +#### TEMP - need to unattach from entities before this can be delete #### +module "pashub_to_ara_s3_write" { source = "../modules/s3_iam_policy" - policy_name = "EnergyAssessmentsWriteS3" - policy_description = "Allow lambdas to write to retrofit energy assessments bucket" + policy_name = "PashubToAraWriteS3" + policy_description = "Allow PasHub to ARA Lambda to write to retrofit energy assessments bucket" bucket_arns = ["arn:aws:s3:::retrofit-energy-assessments-${var.stage}"] actions = ["s3:PutObject", "s3:AbortMultipartUpload"] resource_paths = ["/*"] } -output "energy_assessments_s3_write_arn" { - value = module.energy_assessments_s3_write.policy_arn +output "pashub_to_ara_s3_write_arn" { + value = module.pashub_to_ara_s3_write.policy_arn } ################################################