Upload gzip-compressed MagicPlan JSON to S3 🟥

backend/app/db/models/uploaded_file.py

@@ -17,6 +17,7 @@ class FileTypeEnum(enum.Enum):
     ECMK_SITE_NOTE = "ecmk_site_note"
     ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note"
     ECMK_SURVEY_XML = "ecmk_survey_xml"
+    MAGIC_PLAN_JSON = "magic_plan_json"
 
 
 class FileSourceEnum(enum.Enum):
@@ -24,6 +25,7 @@ class FileSourceEnum(enum.Enum):
     SHAREPOINT = "sharepoint"
     HUBSPOT = "hubspot"
     ECMK = "ecmk"
+    MAGIC_PLAN = "magic_plan"
 
 
 class UploadedFile(Base):

backend/magic_plan/handler.py

@@ -19,7 +19,7 @@ def handler(body: dict[str, Any], context: Any) -> str:
         customer_id=settings.MAGICPLAN_CUSTOMER_ID,
         api_key=settings.MAGICPLAN_API_KEY,
     )
-    plan: Plan = MagicPlanService(client).run(payload.address, payload.uprn)
+    plan: Plan = MagicPlanService(client, s3_bucket="retrofit-energy-assessments-dev").run(payload)
     logger.info("Saved MagicPlan plan uid=%s", plan.uid)
     return plan.uid
 

backend/magic_plan/magic_plan_service.py

@@ -1,3 +1,4 @@
+import gzip
 from typing import Optional
 
 from datatypes.magicplan.api.response import (
@@ -12,23 +13,29 @@ from backend.app.db.connection import db_session
 from backend.app.db.functions.magic_plan_functions import save_plan
 from backend.magic_plan.address_matcher import find_matching_plan
 from backend.magic_plan.magic_plan_client import MagicPlanClient
+from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest
 from utils.logger import setup_logger
+from utils.s3 import save_data_to_s3
 
 logger = setup_logger()
 
 
 class MagicPlanService:
-    def __init__(self, client: MagicPlanClient) -> None:
+    def __init__(self, client: MagicPlanClient, s3_bucket: str) -> None:
         self._client = client
+        self._s3_bucket = s3_bucket
+
+    def run(self, request: MagicPlanTriggerRequest) -> Plan:
+        address = request.address
+        uprn = request.uprn
 
-    def run(self, address: str, uprn: Optional[str] = None) -> Plan:
         if uprn is not None:
             logger.info("MagicPlanService.run uprn=%s", uprn)
 
         plans_response: PlansListResponse = self._client.get_plans()
         matched: Optional[PlanSummary] = find_matching_plan(
             plans_response.plans, address
-        )  # TODO: use address2UPRN instead? or create AddressMatch domain class
+        )
 
         if matched is None:
             raise ValueError(f"No MagicPlan found for address: {address!r}")

backend/magic_plan/tests/test_magic_plan_service.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from unittest.mock import MagicMock, patch
+from unittest.mock import ANY, MagicMock, patch
 
 import pytest
 
@@ -10,9 +10,11 @@ from datatypes.magicplan.domain.models import Plan
 
 from backend.magic_plan.magic_plan_client import MagicPlanClient
 from backend.magic_plan.magic_plan_service import MagicPlanService
+from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest
 
 FIXTURE_DIR = Path(__file__).parents[2] / "magic_plan"
 PLAN_ID = "a7285ed1-878d-47eb-8aa6-85ef9e187516"
+S3_BUCKET = "test-bucket"
 
 
 @pytest.fixture(scope="module")
@@ -45,7 +47,17 @@ def mock_client() -> MagicMock:
 
 
 def _make_service(mock_client: MagicMock) -> MagicPlanService:
-    return MagicPlanService(client=mock_client)
+    return MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET)
+
+
+def _make_request(
+    address: str = "2 Laburnum Way Bromley BR2 8BZ",
+    hubspot_deal_id: str = "deal-123",
+    uprn: str | None = None,
+) -> MagicPlanTriggerRequest:
+    return MagicPlanTriggerRequest(
+        address=address, hubspot_deal_id=hubspot_deal_id, uprn=uprn
+    )
 
 
 # --- no match ---
@@ -57,7 +69,7 @@ def test_run_raises_when_no_plan_found(mock_client: MagicMock) -> None:
     service = _make_service(mock_client)
     # Act / Assert
     with pytest.raises(ValueError, match="No MagicPlan found"):
-        service.run("99 Nowhere Road London SW1A 1AA")
+        service.run(_make_request(address="99 Nowhere Road London SW1A 1AA"))
 
 
 # --- match found ---
@@ -78,8 +90,10 @@ def test_run_fetches_plan_with_matched_id(
         return_value=plan_summary,
     ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch(
         "backend.magic_plan.magic_plan_service.db_session"
+    ), patch(
+        "backend.magic_plan.magic_plan_service.save_data_to_s3"
     ):
-        service.run("2 Laburnum Way Bromley BR2 8BZ")
+        service.run(_make_request())
     # Assert
     mock_client.get_plan.assert_called_once_with(plan_summary.id)
 
@@ -99,8 +113,10 @@ def test_run_returns_mapped_plan(
         return_value=plan_summary,
     ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch(
         "backend.magic_plan.magic_plan_service.db_session"
+    ), patch(
+        "backend.magic_plan.magic_plan_service.save_data_to_s3"
     ):
-        result = service.run("2 Laburnum Way Bromley BR2 8BZ")
+        result = service.run(_make_request())
     # Assert
     assert isinstance(result, Plan)
     assert result.uid == PLAN_ID
@@ -120,8 +136,10 @@ def test_run_calls_save_plan_with_mapped_plan(
         return_value=plan_summary,
     ), patch("backend.magic_plan.magic_plan_service.save_plan") as mock_save, patch(
         "backend.magic_plan.magic_plan_service.db_session"
+    ), patch(
+        "backend.magic_plan.magic_plan_service.save_data_to_s3"
     ):
-        service.run("2 Laburnum Way Bromley BR2 8BZ")
+        service.run(_make_request())
     # Assert — save_plan called with a Plan whose uid matches
     call_args = mock_save.call_args
     saved_plan: Plan = call_args[0][1]
@@ -142,5 +160,39 @@ def test_run_accepts_uprn_without_error(
         return_value=plan_summary,
     ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch(
         "backend.magic_plan.magic_plan_service.db_session"
+    ), patch(
+        "backend.magic_plan.magic_plan_service.save_data_to_s3"
     ):
-        service.run("2 Laburnum Way Bromley BR2 8BZ", uprn="100023336956")
+        service.run(_make_request(uprn="100023336956"))
+
+
+# --- S3 upload ---
+
+
+def test_run_uploads_to_s3_with_uprn_key(
+    mock_client: MagicMock,
+    api_magic_plan: MagicPlanPlan,
+    plan_summary: PlanSummary,
+) -> None:
+    # Arrange
+    mock_client.get_plans.return_value.plans = [plan_summary]
+    mock_client.get_plan.return_value = api_magic_plan
+    mock_client.get_plan_raw.return_value = b'{"raw": "data"}'
+    request = _make_request(uprn="100023336956")
+    service = MagicPlanService(client=mock_client, s3_bucket=S3_BUCKET)
+    with patch(
+        "backend.magic_plan.magic_plan_service.find_matching_plan",
+        return_value=plan_summary,
+    ), patch("backend.magic_plan.magic_plan_service.save_plan"), patch(
+        "backend.magic_plan.magic_plan_service.db_session"
+    ), patch(
+        "backend.magic_plan.magic_plan_service.save_data_to_s3"
+    ) as mock_s3:
+        # Act
+        service.run(request)
+    # Assert
+    mock_s3.assert_called_once_with(
+        ANY,
+        S3_BUCKET,
+        f"documents/uprn/100023336956/magic_plan_{plan_summary.id}.json.gz",
+    )