diff --git a/.devcontainer/asset_list/devcontainer.json b/.devcontainer/asset_list/devcontainer.json index dfa9ba4d..d0c9abbb 100644 --- a/.devcontainer/asset_list/devcontainer.json +++ b/.devcontainer/asset_list/devcontainer.json @@ -9,7 +9,7 @@ // Optional, just makes getting from Downloads (local env) easier "source=${localEnv:HOME},target=/home/vscode,type=bind" ], - "forwardPorts": [8081], + "forwardPorts": ["model-sal:8080"], "customizations": { "vscode": { "extensions": [ diff --git a/.devcontainer/asset_list/docker-compose.yml b/.devcontainer/asset_list/docker-compose.yml index 0568393b..d567162b 100644 --- a/.devcontainer/asset_list/docker-compose.yml +++ b/.devcontainer/asset_list/docker-compose.yml @@ -12,7 +12,10 @@ services: networks: - model-net ports: - - "8081:8080" + # Host port left unspecified so Docker assigns a free one — lets multiple + # worktrees of this repo run at once without colliding. VS Code's + # forwardPorts (in devcontainer.json) forwards container :8080 to your machine. + - "8080" networks: model-net: diff --git a/.devcontainer/backend/devcontainer.json b/.devcontainer/backend/devcontainer.json index 0a78dadf..d5133df3 100644 --- a/.devcontainer/backend/devcontainer.json +++ b/.devcontainer/backend/devcontainer.json @@ -42,9 +42,9 @@ "containerEnv": { "PYTHONFLAGS": "-Xfrozen_modules=off" }, - "forwardPorts": [8000], + "forwardPorts": ["model-backend:8000"], "portsAttributes": { - "8000": { + "model-backend:8000": { "label": "FastAPI", "onAutoForward": "notify" } diff --git a/.devcontainer/backend/docker-compose.yml b/.devcontainer/backend/docker-compose.yml index cf3bb2c0..9b4873c2 100644 --- a/.devcontainer/backend/docker-compose.yml +++ b/.devcontainer/backend/docker-compose.yml @@ -10,7 +10,10 @@ services: USER_GID: ${GID:-1000} command: sleep infinity ports: - - "8000:8000" + # Host port left unspecified so Docker assigns a free one — lets multiple + # worktrees of this repo run at once without colliding. VS Code's + # forwardPorts (in devcontainer.json) forwards container :8000 to your machine. + - "8000" volumes: - ../../:/workspaces/model - ~/.gitconfig:/home/vscode/.gitconfig:ro @@ -30,7 +33,10 @@ services: image: postgres:17.4 restart: unless-stopped ports: - - 5432:5432 + # Dynamic host port (see model-backend above) so a second worktree's db + # doesn't collide on 5432. Reach it from inside the container via host + # "db:5432"; from your machine, use the forwarded port VS Code reports. + - "5432" environment: - PGDATABASE=tech_team_local_db - POSTGRES_USER=postgres diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 73660bb5..e1e9b3c5 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -631,7 +631,7 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/magic_plan/handler/Dockerfile + dockerfile_path: applications/magic_plan/handler/Dockerfile build_context: . secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index 6fe947ce..329a1319 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -119,7 +119,7 @@ jobs: magic_plan_smoke_test: uses: ./.github/workflows/_smoke_test_lambda.yml with: - dockerfile_path: backend/magic_plan/handler/Dockerfile + dockerfile_path: applications/magic_plan/handler/Dockerfile build_context: . service_name: magic-plan diff --git a/backend/app/db/functions/tests/__init__.py b/applications/magic_plan/__init__.py similarity index 100% rename from backend/app/db/functions/tests/__init__.py rename to applications/magic_plan/__init__.py diff --git a/backend/magic_plan/address_matcher.py b/applications/magic_plan/address_matcher.py similarity index 95% rename from backend/magic_plan/address_matcher.py rename to applications/magic_plan/address_matcher.py index 3477c535..68f2474b 100644 --- a/backend/magic_plan/address_matcher.py +++ b/applications/magic_plan/address_matcher.py @@ -1,7 +1,7 @@ import re from typing import Optional -from datatypes.magicplan.api.response import PlanSummary +from domain.magicplan.api.response import PlanSummary _UK_POSTCODE_RE = re.compile(r"[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}", re.IGNORECASE) diff --git a/applications/magic_plan/handler.py b/applications/magic_plan/handler.py new file mode 100644 index 00000000..13bb59c4 --- /dev/null +++ b/applications/magic_plan/handler.py @@ -0,0 +1,50 @@ +import os +from typing import Any, Optional + +import boto3 + +from infrastructure.magic_plan.config import MagicPlanConfig +from infrastructure.magic_plan.magic_plan_client import MagicPlanClient +from infrastructure.s3.s3_client import S3Client +from orchestration.magic_plan_orchestrator import MagicPlanOrchestrator +from applications.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest +from domain.magicplan.models import Plan +from utilities.aws_lambda.subtask_handler import subtask_handler +from utilities.logger import setup_logger + +logger = setup_logger() + + +@subtask_handler() +def handler(body: dict[str, Any], context: Any) -> Optional[str]: + config = MagicPlanConfig.from_env(os.environ) + payload = MagicPlanTriggerRequest.model_validate(body) + client = MagicPlanClient( + customer_id=config.customer_id, + api_key=config.api_key, + ) + + boto3_client: Any = boto3.client # type: ignore + boto_s3: Any = boto3_client("s3") + s3_client = S3Client( + boto_s3_client=boto_s3, bucket="retrofit-energy-assessments-dev" + ) + # TODO: read s3_bucket from env var so staging/prod use the correct bucket + + plan: Optional[Plan] = MagicPlanOrchestrator(client, s3_client).run(payload) + if plan: + logger.info("Saved MagicPlan plan uid=%s", plan.uid) + return plan.uid + + return None + + +if __name__ == "__main__": + event = { + "Records": [ + { + "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ", "hubspot_deal_id": "local-test-deal"}', + } + ] + } + handler(event, None) diff --git a/applications/magic_plan/handler/Dockerfile b/applications/magic_plan/handler/Dockerfile new file mode 100644 index 00000000..b51da9ec --- /dev/null +++ b/applications/magic_plan/handler/Dockerfile @@ -0,0 +1,17 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/magic_plan/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utilities/ utilities/ +COPY backend/ backend/ +COPY applications/ applications/ +COPY domain/ domain/ +COPY datatypes/ datatypes/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY infrastructure/ infrastructure/ + +CMD ["applications.magic_plan.handler.handler"] diff --git a/backend/magic_plan/handler/requirements.txt b/applications/magic_plan/handler/requirements.txt similarity index 100% rename from backend/magic_plan/handler/requirements.txt rename to applications/magic_plan/handler/requirements.txt diff --git a/backend/magic_plan/local_handler/docker-compose.yml b/applications/magic_plan/local_handler/docker-compose.yml similarity index 71% rename from backend/magic_plan/local_handler/docker-compose.yml rename to applications/magic_plan/local_handler/docker-compose.yml index 5a42d259..44b448bb 100644 --- a/backend/magic_plan/local_handler/docker-compose.yml +++ b/applications/magic_plan/local_handler/docker-compose.yml @@ -4,7 +4,7 @@ services: ecmk-fetcher-lambda: build: context: ../../../ - dockerfile: backend/magic_plan/handler/Dockerfile + dockerfile: applications/magic_plan/handler/Dockerfile ports: - "9000:8080" env_file: diff --git a/applications/magic_plan/local_handler/invoke_local_lambda.py b/applications/magic_plan/local_handler/invoke_local_lambda.py new file mode 100644 index 00000000..9221b6bf --- /dev/null +++ b/applications/magic_plan/local_handler/invoke_local_lambda.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import json +import requests + +HOST = "localhost" +PORT = "9000" + +LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations" + +payload = { + "Records": [ + { + "messageId": "test-message-id", + "body": json.dumps( + { + # "task_id": "00000000-0000-0000-0000-000000000001", + # "sub_task_id": "00000000-0000-0000-0000-000000000002", + "address": "63 Dunkery Road, Wythenshawe, M22 0WR | EPC", + "hubspot_deal_id": "501851906250", + } + # { + # "task_id": "00000000-0000-0000-0000-000000000001", + # "sub_task_id": "00000000-0000-0000-0000-000000000002", + # "address": "33 Wallaby Way, Sydney", + # "hubspot_deal_id": "123456789", + # } + ), + } + ] +} + +response = requests.post(LAMBDA_URL, json=payload) + +print("Status code:", response.status_code) +print("Response:") +print(response.text) diff --git a/applications/magic_plan/local_handler/invoke_local_orchestrator.py b/applications/magic_plan/local_handler/invoke_local_orchestrator.py new file mode 100644 index 00000000..2a1e561e --- /dev/null +++ b/applications/magic_plan/local_handler/invoke_local_orchestrator.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Run MagicPlanOrchestrator directly, bypassing @subtask_handler. + +Loads credentials from the repo-root .env file so no DB task/subtask rows +are needed. +""" + +import os +import sys +from pathlib import Path + +import boto3 +from dotenv import load_dotenv + +from utilities.logger import setup_logger + +setup_logger() + +REPO_ROOT = Path(__file__).resolve().parents[3] +load_dotenv(REPO_ROOT / ".env") + +sys.path.insert(0, str(REPO_ROOT)) + +from infrastructure.magic_plan.config import MagicPlanConfig +from infrastructure.magic_plan.magic_plan_client import MagicPlanClient +from infrastructure.s3.s3_client import S3Client +from orchestration.magic_plan_orchestrator import MagicPlanOrchestrator +from applications.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest + +ADDRESS = "20 Larch Way, Bromley, BR2 8DU | Retrofit Assessment" +HUBSPOT_DEAL_ID = "500328089833" +# ADDRESS = "33 Wallaby Way, Sydney" +# HUBSPOT_DEAL_ID = "123456789" + +config = MagicPlanConfig.from_env(os.environ) +client = MagicPlanClient(customer_id=config.customer_id, api_key=config.api_key) + +boto3_client = boto3.client # type: ignore[attr-defined] +s3_client = S3Client( + boto_s3_client=boto3_client("s3"), + bucket="retrofit-energy-assessments-dev", +) + +request = MagicPlanTriggerRequest(address=ADDRESS, hubspot_deal_id=HUBSPOT_DEAL_ID) + +print(f"Running MagicPlanOrchestrator for: {ADDRESS!r}") +plan = MagicPlanOrchestrator(client, s3_client).run(request) +print(f"Done") diff --git a/backend/magic_plan/magic_plan_trigger_request.py b/applications/magic_plan/magic_plan_trigger_request.py similarity index 100% rename from backend/magic_plan/magic_plan_trigger_request.py rename to applications/magic_plan/magic_plan_trigger_request.py diff --git a/backend/magic_plan/magicplan_api_plans_response_example.json b/applications/magic_plan/magicplan_api_plans_response_example.json similarity index 100% rename from backend/magic_plan/magicplan_api_plans_response_example.json rename to applications/magic_plan/magicplan_api_plans_response_example.json diff --git a/backend/magic_plan/__init__.py b/applications/magic_plan/tests/__init__.py similarity index 100% rename from backend/magic_plan/__init__.py rename to applications/magic_plan/tests/__init__.py diff --git a/backend/magic_plan/tests/test_address_matcher.py b/applications/magic_plan/tests/test_address_matcher.py similarity index 95% rename from backend/magic_plan/tests/test_address_matcher.py rename to applications/magic_plan/tests/test_address_matcher.py index 347a49ef..f5d1961b 100644 --- a/backend/magic_plan/tests/test_address_matcher.py +++ b/applications/magic_plan/tests/test_address_matcher.py @@ -1,5 +1,5 @@ -from datatypes.magicplan.api.response import PlanSummary -from backend.magic_plan.address_matcher import find_matching_plan, _extract_postcode +from domain.magicplan.api.response import PlanSummary +from applications.magic_plan.address_matcher import find_matching_plan, _extract_postcode def _make_plan( diff --git a/backend/app/db/functions/magic_plan_functions.py b/backend/app/db/functions/magic_plan_functions.py deleted file mode 100644 index 143e4172..00000000 --- a/backend/app/db/functions/magic_plan_functions.py +++ /dev/null @@ -1,143 +0,0 @@ -from typing import Any, cast - -from sqlalchemy import delete, select -from sqlalchemy.dialects.postgresql import insert as pg_insert -from sqlmodel import Session, col - -from datatypes.magicplan.domain.models import Floor, Plan -from backend.app.db.models.magic_plan import ( - MagicPlanDoorModel, - MagicPlanFloorModel, - MagicPlanPlanModel, - MagicPlanRoomModel, - MagicPlanWindowModel, -) - - -def save_plan(session: Session, plan: Plan, uploaded_file_id: int) -> None: - plan_id: int = _upsert_plan(session, plan, uploaded_file_id) - _delete_children(session, plan_id) - floor_ids: list[int] = _insert_floors(session, plan.floors, plan_id) - room_ids: list[int] = _insert_rooms(session, plan.floors, floor_ids) - _insert_windows_and_doors(session, plan.floors, room_ids) - - -def _upsert_plan(session: Session, plan: Plan, uploaded_file_id: int) -> int: - stmt = ( - pg_insert(MagicPlanPlanModel) - .values( - magic_plan_uid=plan.uid, - name=plan.name, - address=plan.address, - postcode=plan.postcode, - uploaded_file_id=uploaded_file_id, - ) - .on_conflict_do_update( - index_elements=["magic_plan_uid"], - set_={ - "name": plan.name, - "address": plan.address, - "postcode": plan.postcode, - "uploaded_file_id": uploaded_file_id, - }, - ) - .returning(col(MagicPlanPlanModel.id)) - ) - row_id: int = session.execute(stmt).scalar_one() - return row_id - - -def _delete_children(session: Session, plan_id: int) -> None: - floor_subq = ( - select(col(MagicPlanFloorModel.id)) - .where(col(MagicPlanFloorModel.magic_plan_plan_id) == plan_id) - .scalar_subquery() - ) - room_subq = ( - select(col(MagicPlanRoomModel.id)) - .where(col(MagicPlanRoomModel.magic_plan_floor_id).in_(floor_subq)) - .scalar_subquery() - ) - session.execute( - delete(MagicPlanWindowModel).where( - col(MagicPlanWindowModel.magic_plan_room_id).in_(room_subq) - ) - ) - session.execute( - delete(MagicPlanDoorModel).where( - col(MagicPlanDoorModel.magic_plan_room_id).in_(room_subq) - ) - ) - session.execute( - delete(MagicPlanRoomModel).where( - col(MagicPlanRoomModel.magic_plan_floor_id).in_(floor_subq) - ) - ) - session.execute( - delete(MagicPlanFloorModel).where( - col(MagicPlanFloorModel.magic_plan_plan_id) == plan_id - ) - ) - - -def _insert_floors(session: Session, floors: list[Floor], plan_id: int) -> list[int]: - rows: list[dict[str, Any]] = [ - {"magic_plan_plan_id": plan_id, "level": floor.level} for floor in floors - ] - result = session.execute( - pg_insert(MagicPlanFloorModel) - .values(rows) - .returning(col(MagicPlanFloorModel.id)) - ) - return cast(list[int], list(result.scalars().all())) - - -def _insert_rooms( - session: Session, floors: list[Floor], floor_ids: list[int] -) -> list[int]: - rows: list[dict[str, Any]] = [ - { - "magic_plan_floor_id": floor_id, - "name": room.name, - "width_m": room.width_m, - "length_m": room.length_m, - "area_m2": room.area_m2, - } - for floor, floor_id in zip(floors, floor_ids) - for room in floor.rooms - ] - result = session.execute( - pg_insert(MagicPlanRoomModel).values(rows).returning(col(MagicPlanRoomModel.id)) - ) - return cast(list[int], list(result.scalars().all())) - - -def _insert_windows_and_doors( - session: Session, floors: list[Floor], room_ids: list[int] -) -> None: - all_rooms = [room for floor in floors for room in floor.rooms] - - window_rows: list[dict[str, Any]] = [ - { - "magic_plan_room_id": room_id, - "width_m": window.width_m, - "height_m": window.height_m, - "area_m2": window.area_m2, - "opening_type": window.opening_type, - } - for room, room_id in zip(all_rooms, room_ids) - for window in room.windows - ] - door_rows: list[dict[str, Any]] = [ - { - "magic_plan_room_id": room_id, - "width_mm": door.width_mm, - } - for room, room_id in zip(all_rooms, room_ids) - for door in room.doors - ] - - if window_rows: - session.execute(pg_insert(MagicPlanWindowModel).values(window_rows)) - if door_rows: - session.execute(pg_insert(MagicPlanDoorModel).values(door_rows)) diff --git a/backend/app/db/functions/tests/conftest.py b/backend/app/db/functions/tests/conftest.py deleted file mode 100644 index 3f97e92b..00000000 --- a/backend/app/db/functions/tests/conftest.py +++ /dev/null @@ -1,41 +0,0 @@ -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlmodel import SQLModel - -import backend.app.db.models.magic_plan # noqa: F401 — registers MagicPlan models with SQLModel.metadata - -# TODO: promote to backend/app/db/conftest.py once a second DB-touching test directory appears under this tree - - -@pytest.fixture(scope="function") -def engine(postgresql): - connection_string = ( - f"postgresql+psycopg://" - f"{postgresql.info.user}:" - f"{postgresql.info.password}@" - f"{postgresql.info.host}:" - f"{postgresql.info.port}/" - f"{postgresql.info.dbname}" - ) - - engine = create_engine(connection_string) - SQLModel.metadata.create_all(engine) - - yield engine - - SQLModel.metadata.drop_all(engine) - engine.dispose() - - -@pytest.fixture(scope="function") -def db_session(engine): - connection = engine.connect() - transaction = connection.begin() - session = sessionmaker(bind=connection)() - - yield session - - session.close() - transaction.rollback() - connection.close() diff --git a/backend/app/db/functions/tests/test_magic_plan_functions.py b/backend/app/db/functions/tests/test_magic_plan_functions.py deleted file mode 100644 index 0b93685c..00000000 --- a/backend/app/db/functions/tests/test_magic_plan_functions.py +++ /dev/null @@ -1,115 +0,0 @@ -import json -from pathlib import Path - -import pytest -from sqlalchemy import func, select -from sqlalchemy.orm import Session -from sqlmodel import SQLModel - -from datatypes.magicplan.api.response import MagicPlanPlan -from datatypes.magicplan.domain.mapper import map_plan -from datatypes.magicplan.domain.models import Plan - -from backend.app.db.functions.magic_plan_functions import save_plan -from backend.app.db.models.magic_plan import ( - MagicPlanDoorModel, - MagicPlanFloorModel, - MagicPlanPlanModel, - MagicPlanRoomModel, - MagicPlanWindowModel, -) - -FIXTURE_DIR = Path(__file__).parents[4] / "magic_plan" - - -@pytest.fixture(scope="module") -def domain_plan() -> Plan: - data = json.loads( - (FIXTURE_DIR / "magicplan_api_plan_response_example.json").read_text() - ) - return map_plan(MagicPlanPlan.model_validate(data["data"])) - - -def _count(session: Session, model: type[SQLModel]) -> int: - return session.execute(select(func.count()).select_from(model)).scalar_one() - - -def test_plan_row_present_after_save(db_session: Session, domain_plan: Plan) -> None: - # Act - save_plan(db_session, domain_plan, 1) - # Assert - assert _count(db_session, MagicPlanPlanModel) == 1 - - -def test_floor_count_matches_domain(db_session: Session, domain_plan: Plan) -> None: - # Arrange - expected = len(domain_plan.floors) - # Act - save_plan(db_session, domain_plan, 1) - # Assert - assert _count(db_session, MagicPlanFloorModel) == expected - - -def test_room_count_matches_domain(db_session: Session, domain_plan: Plan) -> None: - # Arrange - expected = sum(len(f.rooms) for f in domain_plan.floors) - # Act - save_plan(db_session, domain_plan, 1) - # Assert - assert _count(db_session, MagicPlanRoomModel) == expected - - -def test_window_count_matches_domain(db_session: Session, domain_plan: Plan) -> None: - # Arrange - expected = sum(len(r.windows) for f in domain_plan.floors for r in f.rooms) - # Act - save_plan(db_session, domain_plan, 1) - # Assert - assert _count(db_session, MagicPlanWindowModel) == expected - - -def test_door_count_matches_domain(db_session: Session, domain_plan: Plan) -> None: - # Arrange - expected = sum(len(r.doors) for f in domain_plan.floors for r in f.rooms) - # Act - save_plan(db_session, domain_plan, 1) - # Assert - assert _count(db_session, MagicPlanDoorModel) == expected - - -def test_save_plan_idempotent(db_session: Session, domain_plan: Plan) -> None: - # Act — call twice within the same session - save_plan(db_session, domain_plan, 1) - save_plan(db_session, domain_plan, 1) - # Assert — same row counts as a single call - assert _count(db_session, MagicPlanPlanModel) == 1 - assert _count(db_session, MagicPlanFloorModel) == len(domain_plan.floors) - assert _count(db_session, MagicPlanRoomModel) == sum( - len(f.rooms) for f in domain_plan.floors - ) - assert _count(db_session, MagicPlanWindowModel) == sum( - len(r.windows) for f in domain_plan.floors for r in f.rooms - ) - assert _count(db_session, MagicPlanDoorModel) == sum( - len(r.doors) for f in domain_plan.floors for r in f.rooms - ) - - -def test_uploaded_file_id_stored_after_save(db_session: Session, domain_plan: Plan) -> None: - # Act - save_plan(db_session, domain_plan, 1) - # Assert - row = db_session.execute(select(MagicPlanPlanModel)).scalar_one() - assert row.uploaded_file_id == 1 - - -def test_save_plan_updates_uploaded_file_id_on_reingest( - db_session: Session, domain_plan: Plan -) -> None: - # Arrange - save_plan(db_session, domain_plan, 1) - # Act - save_plan(db_session, domain_plan, 2) - # Assert - row = db_session.execute(select(MagicPlanPlanModel)).scalar_one() - assert row.uploaded_file_id == 2 diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 2935f2bf..5104f377 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -82,6 +82,11 @@ class HubspotDealData(SQLModel, table=True): domna_survey_required: Optional[bool] = Field(default=None) domna_survey_date: Optional[datetime] = Field(default=None) + date_booking_made: Optional[datetime] = Field(default=None) + last_contact_date: Optional[datetime] = Field(default=None) + last_outbound_call: Optional[datetime] = Field(default=None) + last_outbound_email: Optional[datetime] = Field(default=None) + created_at: Optional[datetime] = Field( sa_column=Column( DateTime(timezone=True), diff --git a/backend/app/db/models/magic_plan.py b/backend/app/db/models/magic_plan.py deleted file mode 100644 index 77ca52fd..00000000 --- a/backend/app/db/models/magic_plan.py +++ /dev/null @@ -1,53 +0,0 @@ -from typing import Optional - -from sqlmodel import Field, SQLModel - - -class MagicPlanPlanModel(SQLModel, table=True): - __tablename__ = "magic_plan_plan" - - id: Optional[int] = Field(default=None, primary_key=True) - magic_plan_uid: Optional[str] = Field(default=None, unique=True, index=True) - name: Optional[str] = None - address: Optional[str] = None - postcode: Optional[str] = None - uploaded_file_id: Optional[int] = Field(default=None) - - -class MagicPlanFloorModel(SQLModel, table=True): - __tablename__ = "magic_plan_floor" - - id: Optional[int] = Field(default=None, primary_key=True) - magic_plan_plan_id: int = Field(foreign_key="magic_plan_plan.id") - level: Optional[int] = None - - -class MagicPlanRoomModel(SQLModel, table=True): - __tablename__ = "magic_plan_room" - - id: Optional[int] = Field(default=None, primary_key=True) - magic_plan_floor_id: int = Field(foreign_key="magic_plan_floor.id") - name: Optional[str] = None - width_m: Optional[float] = None - length_m: Optional[float] = None - area_m2: Optional[float] = None - - -class MagicPlanWindowModel(SQLModel, table=True): - __tablename__ = "magic_plan_window" - - id: Optional[int] = Field(default=None, primary_key=True) - magic_plan_room_id: int = Field(foreign_key="magic_plan_room.id") - width_m: Optional[float] = None - height_m: Optional[float] = None - area_m2: Optional[float] = None - opening_type: Optional[str] = None - - -class MagicPlanDoorModel(SQLModel, table=True): - __tablename__ = "magic_plan_door" - - id: Optional[int] = Field(default=None, primary_key=True) - magic_plan_room_id: int = Field(foreign_key="magic_plan_room.id") - width_mm: Optional[float] = None - type: Optional[str] = None diff --git a/backend/app/db/models/tests/test_magic_plan_models.py b/backend/app/db/models/tests/test_magic_plan_models.py index 0830b184..6371168c 100644 --- a/backend/app/db/models/tests/test_magic_plan_models.py +++ b/backend/app/db/models/tests/test_magic_plan_models.py @@ -1,4 +1,8 @@ -from backend.app.db.models.magic_plan import ( +from typing import Any, cast + +import sqlalchemy as sa + +from infrastructure.postgres.magic_plan_tables import ( MagicPlanDoorModel, MagicPlanFloorModel, MagicPlanPlanModel, @@ -6,6 +10,10 @@ from backend.app.db.models.magic_plan import ( MagicPlanWindowModel, ) + +def _table(model: type[Any]) -> sa.Table: + return cast(sa.Table, getattr(model, "__table__")) + # --- MagicPlanPlan --- @@ -14,20 +22,17 @@ def test_plan_table_name() -> None: def test_plan_has_magic_plan_uid_column() -> None: - assert "magic_plan_uid" in MagicPlanPlanModel.__table__.columns + assert "magic_plan_uid" in _table(MagicPlanPlanModel).columns def test_plan_magic_plan_uid_is_unique() -> None: - col = MagicPlanPlanModel.__table__.columns["magic_plan_uid"] - assert ( - any( - c.unique - for c in MagicPlanPlanModel.__table__.constraints - if hasattr(c, "columns") - and "magic_plan_uid" in [cc.name for cc in c.columns] - ) - or col.unique + t = _table(MagicPlanPlanModel) + col = t.columns["magic_plan_uid"] + has_unique_constraint = any( + isinstance(c, sa.UniqueConstraint) and "magic_plan_uid" in c.columns + for c in t.constraints ) + assert has_unique_constraint or col.unique def test_plan_instantiation() -> None: @@ -47,7 +52,7 @@ def test_floor_table_name() -> None: def test_floor_fk_column_name() -> None: - assert "magic_plan_plan_id" in MagicPlanFloorModel.__table__.columns + assert "magic_plan_plan_id" in _table(MagicPlanFloorModel).columns def test_floor_has_level() -> None: @@ -63,11 +68,11 @@ def test_room_table_name() -> None: def test_room_fk_column_name() -> None: - assert "magic_plan_floor_id" in MagicPlanRoomModel.__table__.columns + assert "magic_plan_floor_id" in _table(MagicPlanRoomModel).columns def test_room_has_measurement_columns() -> None: - cols = MagicPlanRoomModel.__table__.columns + cols = _table(MagicPlanRoomModel).columns assert "width_m" in cols assert "length_m" in cols assert "area_m2" in cols @@ -89,15 +94,14 @@ def test_window_table_name() -> None: def test_window_fk_column_name() -> None: - assert "magic_plan_room_id" in MagicPlanWindowModel.__table__.columns + assert "magic_plan_room_id" in _table(MagicPlanWindowModel).columns def test_window_has_measurement_columns() -> None: - cols = MagicPlanWindowModel.__table__.columns + cols = _table(MagicPlanWindowModel).columns assert "width_m" in cols assert "height_m" in cols assert "area_m2" in cols - assert "opening_type" in cols def test_window_instantiation() -> None: @@ -106,9 +110,8 @@ def test_window_instantiation() -> None: width_m=1.4, height_m=1.2, area_m2=1.68, - opening_type="casement", ) - assert window.opening_type == "casement" + assert window.width_m == 1.4 # --- MagicPlanDoor --- @@ -119,16 +122,16 @@ def test_door_table_name() -> None: def test_door_fk_column_name() -> None: - assert "magic_plan_room_id" in MagicPlanDoorModel.__table__.columns + assert "magic_plan_room_id" in _table(MagicPlanDoorModel).columns def test_door_has_width_mm_and_type() -> None: - cols = MagicPlanDoorModel.__table__.columns + cols = _table(MagicPlanDoorModel).columns assert "width_mm" in cols assert "type" in cols def test_door_instantiation() -> None: - door = MagicPlanDoorModel(magic_plan_room_id=1, width_mm=0.79, type="hinged") - assert door.width_mm == 0.79 + door = MagicPlanDoorModel(magic_plan_room_id=1, width_mm=790.0, type="hinged") + assert door.width_mm == 790.0 assert door.type == "hinged" diff --git a/backend/magic_plan/handler.py b/backend/magic_plan/handler.py deleted file mode 100644 index e7dc6484..00000000 --- a/backend/magic_plan/handler.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Any - -from backend.app.config import get_settings -from backend.magic_plan.magic_plan_client import MagicPlanClient -from backend.magic_plan.magic_plan_service import MagicPlanService -from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest -from datatypes.magicplan.domain.models import Plan -from backend.app.db.models.tasks import SourceEnum -from backend.utils.subtasks import task_handler -from utils.logger import setup_logger - -logger = setup_logger() - - -@task_handler(task_source="magic_plan", source=SourceEnum.HUBSPOT_DEAL) -def handler(body: dict[str, Any], context: Any) -> str: - settings = get_settings() - payload = MagicPlanTriggerRequest.model_validate(body) - client = MagicPlanClient( - customer_id=settings.MAGICPLAN_CUSTOMER_ID, - api_key=settings.MAGICPLAN_API_KEY, - ) - # TODO: read s3_bucket from env var so staging/prod use the correct bucket - plan: Plan = MagicPlanService( - client, s3_bucket="retrofit-energy-assessments-dev" - ).run(payload) - logger.info("Saved MagicPlan plan uid=%s", plan.uid) - return plan.uid - - -if __name__ == "__main__": - event = { - "Records": [ - { - "body": '{"address": "2 Laburnum Way Bromley BR2 8BZ", "hubspot_deal_id": "local-test-deal"}', - } - ] - } - handler(event, None) diff --git a/backend/magic_plan/handler/Dockerfile b/backend/magic_plan/handler/Dockerfile deleted file mode 100644 index ffd85c02..00000000 --- a/backend/magic_plan/handler/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM public.ecr.aws/lambda/python:3.11 - -WORKDIR /var/task - -COPY backend/magic_plan/handler/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY utils/ utils/ -COPY backend/ backend/ -COPY datatypes/ datatypes/ - -CMD ["backend.magic_plan.handler.handler"] diff --git a/backend/magic_plan/local_handler/invoke_local_lambda.py b/backend/magic_plan/local_handler/invoke_local_lambda.py deleted file mode 100644 index 146951fe..00000000 --- a/backend/magic_plan/local_handler/invoke_local_lambda.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import json -import requests - -HOST = "localhost" -PORT = "9000" - -LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations" - -payload = { - "Records": [ - { - "messageId": "test-message-id", - "body": json.dumps( - # { - # "address": "2 Laburnum Way, Rombley, BR2 8BZ | Retrofit Assessment", - # "hubspot_deal_id": "500262906061", - # } - {"address": "33 Wallaby Way, Sydney", "hubspot_deal_id": "123456789"} - ), - } - ] -} - -response = requests.post(LAMBDA_URL, json=payload) - -print("Status code:", response.status_code) -print("Response:") -print(response.text) diff --git a/backend/magic_plan/magic_plan_service.py b/backend/magic_plan/magic_plan_service.py deleted file mode 100644 index 8a75c716..00000000 --- a/backend/magic_plan/magic_plan_service.py +++ /dev/null @@ -1,85 +0,0 @@ -import gzip -import json -from datetime import datetime, timezone -from typing import Optional, cast - -from datatypes.magicplan.api.response import MagicPlanPlan, PlanSummary -from datatypes.magicplan.domain.mapper import map_plan -from datatypes.magicplan.domain.models import Plan - -from backend.app.db.connection import db_session -from backend.app.db.functions.magic_plan_functions import save_plan -from backend.app.db.models.uploaded_file import ( - FileSourceEnum, - FileTypeEnum, - UploadedFile, -) -from backend.magic_plan.address_matcher import find_matching_plan -from backend.magic_plan.magic_plan_client import MagicPlanClient -from backend.magic_plan.magic_plan_trigger_request import MagicPlanTriggerRequest -from utils.logger import setup_logger -from utils.s3 import save_data_to_s3 - -logger = setup_logger() - - -class MagicPlanService: - def __init__(self, client: MagicPlanClient, s3_bucket: str) -> None: - self._client = client - self._s3_bucket = s3_bucket - - def run(self, request: MagicPlanTriggerRequest) -> Plan: - address = request.address - uprn = request.uprn - - if uprn is not None: - logger.info("MagicPlanService.run uprn=%s", uprn) - - plans: list[PlanSummary] = self._client.get_plans() - matched: Optional[PlanSummary] = find_matching_plan(plans, address) - - if matched is None: - raise ValueError(f"No MagicPlan found for address: {address!r}") - - raw_bytes: bytes = self._client.get_plan_raw(matched.id) - magic_plan: MagicPlanPlan = MagicPlanPlan.model_validate( - json.loads(raw_bytes)["data"] - ) - plan: Plan = map_plan(magic_plan) - - uploaded_file: UploadedFile = self._upload_raw_plan_json( - plan_id=matched.id, - raw_bytes=raw_bytes, - uprn=uprn, - hubspot_deal_id=request.hubspot_deal_id, - ) - - with db_session() as session: - session.add(uploaded_file) - session.flush() - save_plan(session, plan, cast(int, uploaded_file.id)) - - return plan - - def _upload_raw_plan_json( - self, - plan_id: str, - raw_bytes: bytes, - uprn: Optional[str], - hubspot_deal_id: str, - ) -> UploadedFile: - compressed = gzip.compress(raw_bytes) - if uprn is not None: - s3_key = f"documents/uprn/{uprn}/magic_plan_{plan_id}.json.gz" - else: - s3_key = f"documents/hubspot_deal_id/{hubspot_deal_id}/magic_plan_{plan_id}.json.gz" - save_data_to_s3(compressed, self._s3_bucket, s3_key) - return UploadedFile( - s3_file_bucket=self._s3_bucket, - s3_file_key=s3_key, - s3_upload_timestamp=datetime.now(timezone.utc), - uprn=int(uprn) if uprn is not None else None, - hubspot_deal_id=hubspot_deal_id, - file_source=FileSourceEnum.MAGIC_PLAN.value, - file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, - ) diff --git a/backend/magic_plan/magicplan_api_plan_response_example.json b/backend/magic_plan/magicplan_api_plan_response_example.json deleted file mode 100644 index d76b3540..00000000 --- a/backend/magic_plan/magicplan_api_plan_response_example.json +++ /dev/null @@ -1,136742 +0,0 @@ -{ - "message": "OK", - "data": { - "plan": { - "id": "a7285ed1-878d-47eb-8aa6-85ef9e187516", - "project_id": "9f8f3208-0f04-466f-9c4c-e776532183c8", - "name": "2, Br2 8bz", - "address": { - "street": "2 Laburnum Way", - "street_number": null, - "postal_code": "BR2 8BZ", - "city": "Bromley", - "country": "GB", - "longitude": 0.0616749, - "latitude": 51.3835182 - }, - "creation_date": "2026-04-28T08:32:58+00:00", - "update_date": "2026-04-29T14:58:54+00:00", - "thumbnail_url": "https:\/\/s3.amazonaws.com\/prod.plans.sensopia.com\/a7285ed1-878d-47eb-8aa6-85ef9e187516\/plan.thumb", - "public_url": "https:\/\/cloud.magicplan.app\/plan\/a7285ed1-878d-47eb-8aa6-85ef9e187516", - "cloud_url": "https:\/\/cloud.magicplan.app\/projects\/a7285ed1-878d-47eb-8aa6-85ef9e187516", - "3d_url": "https:\/\/3d.magicplan.app\/#embed\/?key=YzBkMTQyZDRlY2E5MmEzMWQ4NWE1NWJmMGE4OTQ5ZjMwOTNlZjcwNjhkN2U4ODg5ZDZiMDI1OTRkNWU5ZTY0N%2B9n3Xg%2FF422BetMnabb%2FwQI3XiEQbNltioOXI05WueYapFlJvuxgPLnzxjLI1eFcsii6s7vRgs71gHD1LPsSBcNGjF424hTkMCt9hxbCryf", - "workgroup_id": "677d01685458a", - "team_id": null, - "created_by": { - "id": "49c5fd0d-5031-4a7d-aa59-3cc1b64d18aa", - "firstname": null, - "lastname": null, - "email": "sebastian@osmosis-acd.com" - } - }, - "plan_detail": { - "magicplan_format_xml": "\n2026-04-24<\/value>2.134<\/value>100<\/value>0<\/value>0<\/value><\/values>Ground Floor<\/name>2.450007<\/value>Total m2 =1.196 yd\u00b2 <\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>outdoors<\/value>m3<\/value><\/values><\/symbolInstance>annotations<\/value>3<\/value>left<\/value>M2 - 44.19\nHeight - 2.43\nHLP - 20.56\nPWL - 6.12<\/value>top<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>annotations<\/value>7.22m<\/value><\/values><\/symbolInstance>annotations<\/value>6.12m<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.026217<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>0.905517<\/value>1.20394<\/value><\/values><\/symbolInstance>0.496099<\/value>0.241025<\/value>plumbing<\/value>0.682423<\/value>0.241025<\/value>0.454712<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.014143<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.963297<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>hvac<\/value>m3<\/value>0.1500<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.014739<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>0.985417<\/value>1.099043<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>0.911601<\/value>1.123649<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.057803<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>hvac<\/value>m3<\/value>0.1500<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.942764<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>0.968262<\/value>1.063659<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.014143<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.133701<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.014739<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.942764<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>0.867147<\/value>1.202909<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>hvac<\/value>m3<\/value>0.1500<\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>electrical<\/value>m3<\/value>1.1<\/value><\/values><\/symbolInstance>structure<\/value>0<\/value>1<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>2.057803<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.993003<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.0<\/value><\/values><\/symbolInstance>m<\/value>1<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.0<\/value><\/values><\/symbolInstance>m<\/value>0<\/value>m<\/value>m2<\/value>doors<\/value>m3<\/value>1.963297<\/value><\/values><\/symbolInstance>677d01685458a<\/value><\/values><\/symbolInstance>m<\/value>m<\/value>m2<\/value>hvac<\/value>m3<\/value>0.1500<\/value><\/values><\/symbolInstance>2.450007<\/value>0.5x1.2 (x2)\n<\/value>2500 (x2)<\/value>Between.15.and.30.degrees<\/value>0<\/value>9.X.6<\/value>0<\/value>0<\/value>9<\/value><\/values><\/floorRoom>