From 7497865fb2f37ba8a954c9e52d20096a534acdc6 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 8 Jun 2026 15:44:19 +0000 Subject: [PATCH 01/18] =?UTF-8?q?Retrieve=20plan=20by=20uploaded=5Ffile=5F?= =?UTF-8?q?id=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../magic_plan_postgres_repository.py | 253 +++++++++--------- 1 file changed, 129 insertions(+), 124 deletions(-) diff --git a/repositories/magic_plan/magic_plan_postgres_repository.py b/repositories/magic_plan/magic_plan_postgres_repository.py index c2df041e..f8ff123c 100644 --- a/repositories/magic_plan/magic_plan_postgres_repository.py +++ b/repositories/magic_plan/magic_plan_postgres_repository.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Optional, cast +from typing import Any, NamedTuple, Optional, cast from sqlalchemy import delete, select from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -27,148 +27,153 @@ from infrastructure.postgres.magic_plan_tables import ( from repositories.magic_plan.magic_plan_repository import MagicPlanRepository +class _Rows(NamedTuple): + floors: list[MagicPlanFloorModel] + rooms: list[MagicPlanRoomModel] + windows: list[MagicPlanWindowModel] + doors: list[MagicPlanDoorModel] + win_vents: list[MagicPlanWindowVentilationModel] + door_vents: list[MagicPlanDoorVentilationModel] + + +def _build_windows( + rows: list[MagicPlanWindowModel], + vents: list[MagicPlanWindowVentilationModel], +) -> dict[int, list[Window]]: + vent_by_id = {wv.magic_plan_window_id: wv for wv in vents} + result: dict[int, list[Window]] = {} + for row in rows: + wv = vent_by_id.get(cast(int, row.id)) + result.setdefault(row.magic_plan_room_id, []).append( + Window( + width_m=cast(float, row.width_m), + height_m=cast(float, row.height_m), + area_m2=cast(float, row.area_m2), + ventilation=WindowVentilation( + opening_type=wv.opening_type, + num_openings=wv.num_openings, + pct_openable=wv.pct_openable, + trickle_vent_area_mm2=wv.trickle_vent_area_mm2, + num_trickle_vents=wv.num_trickle_vents, + ) if wv else None, + ) + ) + return result + + +def _build_doors( + rows: list[MagicPlanDoorModel], + vents: list[MagicPlanDoorVentilationModel], +) -> dict[int, list[Door]]: + vent_by_id = {dv.magic_plan_door_id: dv for dv in vents} + result: dict[int, list[Door]] = {} + for row in rows: + dv = vent_by_id.get(cast(int, row.id)) + result.setdefault(row.magic_plan_room_id, []).append( + Door( + width_mm=cast(float, row.width_mm), + height_mm=cast(float, row.height_mm), + ventilation=DoorVentilation(undercut_mm=dv.undercut_mm) if dv else None, + ) + ) + return result + + +def _build_rooms( + rows: list[MagicPlanRoomModel], + windows_by_room: dict[int, list[Window]], + doors_by_room: dict[int, list[Door]], +) -> dict[int, list[Room]]: + result: dict[int, list[Room]] = {} + for row in rows: + room_id = cast(int, row.id) + result.setdefault(row.magic_plan_floor_id, []).append( + Room( + name=cast(str, row.name), + width_m=cast(float, row.width_m), + length_m=cast(float, row.length_m), + area_m2=cast(float, row.area_m2), + windows=windows_by_room.get(room_id, []), + doors=doors_by_room.get(room_id, []), + ) + ) + return result + + class MagicPlanPostgresRepository(MagicPlanRepository): def __init__(self, session: Session) -> None: self._session = session def get_plan_by_uploaded_file_id(self, uploaded_file_id: int) -> Optional[Plan]: - plan_row = self._session.execute( # pyright: ignore[reportDeprecated] + plan_row = self._fetch_one( select(MagicPlanPlanModel).where( col(MagicPlanPlanModel.uploaded_file_id) == uploaded_file_id ) - ).scalars().one_or_none() + ) if plan_row is None: return None - plan_id = cast(int, plan_row.id) - - floor_rows = list( - self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanFloorModel).where( - col(MagicPlanFloorModel.magic_plan_plan_id) == plan_id - ) - ).scalars().all() - ) - floor_ids = [cast(int, f.id) for f in floor_rows] - - room_rows = list( - self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanRoomModel).where( - col(MagicPlanRoomModel.magic_plan_floor_id).in_(floor_ids) - ) - ).scalars().all() - ) - room_ids = [cast(int, r.id) for r in room_rows] - - window_rows = list( - self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanWindowModel).where( - col(MagicPlanWindowModel.magic_plan_room_id).in_(room_ids) - ) - ).scalars().all() - ) - window_ids = [cast(int, w.id) for w in window_rows] - - door_rows = list( - self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanDoorModel).where( - col(MagicPlanDoorModel.magic_plan_room_id).in_(room_ids) - ) - ).scalars().all() - ) - door_ids = [cast(int, d.id) for d in door_rows] - - window_vent_by_window_id = { - cast(int, wv.magic_plan_window_id): wv - for wv in self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanWindowVentilationModel).where( - col(MagicPlanWindowVentilationModel.magic_plan_window_id).in_( - window_ids - ) - ) - ).scalars().all() - } - door_vent_by_door_id = { - cast(int, dv.magic_plan_door_id): dv - for dv in self._session.execute( # pyright: ignore[reportDeprecated] - select(MagicPlanDoorVentilationModel).where( - col(MagicPlanDoorVentilationModel.magic_plan_door_id).in_(door_ids) - ) - ).scalars().all() - } - - windows_by_room_id: dict[int, list[Window]] = {} - for w_row, w_id in zip(window_rows, window_ids): - room_id = cast(int, w_row.magic_plan_room_id) - wv_row = window_vent_by_window_id.get(w_id) - ventilation = ( - WindowVentilation( - opening_type=wv_row.opening_type, - num_openings=wv_row.num_openings, - pct_openable=wv_row.pct_openable, - trickle_vent_area_mm2=wv_row.trickle_vent_area_mm2, - num_trickle_vents=wv_row.num_trickle_vents, - ) - if wv_row is not None - else None - ) - windows_by_room_id.setdefault(room_id, []).append( - Window( - width_m=cast(float, w_row.width_m), - height_m=cast(float, w_row.height_m), - area_m2=cast(float, w_row.area_m2), - ventilation=ventilation, - ) - ) - - doors_by_room_id: dict[int, list[Door]] = {} - for d_row, d_id in zip(door_rows, door_ids): - room_id = cast(int, d_row.magic_plan_room_id) - dv_row = door_vent_by_door_id.get(d_id) - ventilation = ( - DoorVentilation(undercut_mm=dv_row.undercut_mm) - if dv_row is not None - else None - ) - doors_by_room_id.setdefault(room_id, []).append( - Door( - width_mm=cast(float, d_row.width_mm), - height_mm=cast(float, d_row.height_mm), - ventilation=ventilation, - ) - ) - - rooms_by_floor_id: dict[int, list[Room]] = {} - for r_row, r_id in zip(room_rows, room_ids): - floor_id = cast(int, r_row.magic_plan_floor_id) - rooms_by_floor_id.setdefault(floor_id, []).append( - Room( - name=cast(str, r_row.name), - width_m=cast(float, r_row.width_m), - length_m=cast(float, r_row.length_m), - area_m2=cast(float, r_row.area_m2), - windows=windows_by_room_id.get(r_id, []), - doors=doors_by_room_id.get(r_id, []), - ) - ) - - floors = [ - Floor( - level=f_row.level, - name=None, - rooms=rooms_by_floor_id.get(cast(int, f_row.id), []), - ) - for f_row in floor_rows - ] + rows = self._fetch_rows(cast(int, plan_row.id)) + windows_by_room = _build_windows(rows.windows, rows.win_vents) + doors_by_room = _build_doors(rows.doors, rows.door_vents) + rooms_by_floor = _build_rooms(rows.rooms, windows_by_room, doors_by_room) return Plan( uid=cast(str, plan_row.magic_plan_uid), name=plan_row.name, address=plan_row.address, postcode=plan_row.postcode, - floors=floors, + floors=[ + Floor(level=f.level, name=None, rooms=rooms_by_floor.get(cast(int, f.id), [])) + for f in rows.floors + ], ) + def _fetch_rows(self, plan_id: int) -> _Rows: + floor_rows: list[MagicPlanFloorModel] = self._fetch_many( + select(MagicPlanFloorModel).where( + col(MagicPlanFloorModel.magic_plan_plan_id) == plan_id + ) + ) + floor_ids = [cast(int, f.id) for f in floor_rows] + room_rows: list[MagicPlanRoomModel] = self._fetch_many( + select(MagicPlanRoomModel).where( + col(MagicPlanRoomModel.magic_plan_floor_id).in_(floor_ids) + ) + ) + room_ids = [cast(int, r.id) for r in room_rows] + window_rows: list[MagicPlanWindowModel] = self._fetch_many( + select(MagicPlanWindowModel).where( + col(MagicPlanWindowModel.magic_plan_room_id).in_(room_ids) + ) + ) + door_rows: list[MagicPlanDoorModel] = self._fetch_many( + select(MagicPlanDoorModel).where( + col(MagicPlanDoorModel.magic_plan_room_id).in_(room_ids) + ) + ) + win_vents: list[MagicPlanWindowVentilationModel] = self._fetch_many( + select(MagicPlanWindowVentilationModel).where( + col(MagicPlanWindowVentilationModel.magic_plan_window_id).in_( + [cast(int, w.id) for w in window_rows] + ) + ) + ) + door_vents: list[MagicPlanDoorVentilationModel] = self._fetch_many( + select(MagicPlanDoorVentilationModel).where( + col(MagicPlanDoorVentilationModel.magic_plan_door_id).in_( + [cast(int, d.id) for d in door_rows] + ) + ) + ) + return _Rows(floor_rows, room_rows, window_rows, door_rows, win_vents, door_vents) + + def _fetch_one(self, stmt: Any) -> Any: + return self._session.execute(stmt).scalars().one_or_none() # pyright: ignore[reportDeprecated] + + def _fetch_many(self, stmt: Any) -> Any: + return list(self._session.execute(stmt).scalars().all()) # pyright: ignore[reportDeprecated] + def save(self, plan: Plan, uploaded_file_id: int) -> None: plan_id = self._upsert_plan(plan, uploaded_file_id) self._delete_children(plan_id) From 209a0c401c8243dbc1033ff8e2dc5580710a0181 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 8 Jun 2026 16:20:05 +0000 Subject: [PATCH 02/18] audit generator application framework --- applications/audit-generator/handler.py | 0 applications/audit-generator/handler/Dockerfile | 0 .../audit-generator/handler/audit_generator_trigger_request.py | 0 applications/audit-generator/handler/requirements.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 applications/audit-generator/handler.py create mode 100644 applications/audit-generator/handler/Dockerfile create mode 100644 applications/audit-generator/handler/audit_generator_trigger_request.py create mode 100644 applications/audit-generator/handler/requirements.txt diff --git a/applications/audit-generator/handler.py b/applications/audit-generator/handler.py new file mode 100644 index 00000000..e69de29b diff --git a/applications/audit-generator/handler/Dockerfile b/applications/audit-generator/handler/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/applications/audit-generator/handler/audit_generator_trigger_request.py b/applications/audit-generator/handler/audit_generator_trigger_request.py new file mode 100644 index 00000000..e69de29b diff --git a/applications/audit-generator/handler/requirements.txt b/applications/audit-generator/handler/requirements.txt new file mode 100644 index 00000000..e69de29b From e5e67c203bde03676a21b712bd96ccbe40645c06 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 8 Jun 2026 16:48:37 +0000 Subject: [PATCH 03/18] temp commit to backup prd in case laptop dies --- backlog/ventilation-audit-generator.md | 99 ++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 backlog/ventilation-audit-generator.md diff --git a/backlog/ventilation-audit-generator.md b/backlog/ventilation-audit-generator.md new file mode 100644 index 00000000..6af37101 --- /dev/null +++ b/backlog/ventilation-audit-generator.md @@ -0,0 +1,99 @@ +# PRD: Ventilation Audit Generator from MagicPlan + +## Problem Statement + +When a surveyor completes a MagicPlan survey for a property, the resulting floor plan data (rooms, windows, doors, ventilation measurements) needs to be transformed into a structured ventilation audit spreadsheet. Currently this transformation is manual — someone must extract plan data and populate a report by hand, which is slow and error-prone. + +## Solution + +An AWS Lambda (`audit-generator`) triggered via SQS receives a HubSpot deal ID, fetches the parsed MagicPlan `Plan` from the database, populates a pre-formatted `.xlsx` template with plan data, uploads the result to S3, and records it in `uploaded_files`. The populated spreadsheet is then accessible to the UI so the user knows an audit file exists for that deal. + +## User Stories + +1. As a coordinator, I want clicking a button in the UI to trigger generation of a ventilation audit spreadsheet, so that I do not have to manually populate it from the floor plan. +2. As a coordinator, I want the audit spreadsheet to be automatically populated with room, window, and door data from the MagicPlan survey, so that the data entry step is eliminated. +3. As a coordinator, I want the system to use a pre-formatted `.xlsx` template when generating the audit, so that conditional formatting and layout are preserved without requiring code changes. +4. As a coordinator, I want the UI to indicate whether a ventilation audit already exists for a deal, so that I avoid triggering duplicate generation unnecessarily. +5. As a coordinator, I want re-triggering generation to overwrite the previous audit file, so that I can regenerate after a corrected survey is uploaded. +6. As an engineer, I want the lambda to raise a clear error if no MagicPlan JSON has been uploaded for the deal, so that misconfigured triggers are diagnosed quickly. +7. As an engineer, I want the lambda to raise a distinct error if a MagicPlan JSON exists but has not yet been parsed into the database, so that timing issues are distinguishable from missing data. +8. As an engineer, I want the generated spreadsheet recorded in `uploaded_files` with a `VENTILATION_AUDIT` file type, so that the UI and other systems can query for its existence. +9. As an engineer, I want the audit template to be resolved from an environment variable, so that different templates can be used in staging and production without a code deploy. +10. As an engineer, I want the lambda to follow the `@subtask_handler()` pattern, so that it integrates with the task orchestration system and benefits from standard error handling and observability. +11. As an engineer, I want the spreadsheet cells to be written via named ranges defined in the template, so that template layout changes do not require code changes. + +## Implementation Decisions + +- **Lambda pattern**: `@subtask_handler()` decorator. Trigger body contains `task_id`, `sub_task_id`, and `hubspot_deal_id`. + +- **MAGIC_PLAN_JSON lookup**: Query `uploaded_files` filtered by `hubspot_deal_id` and `file_type = MAGIC_PLAN_JSON`, ordered by `s3_upload_timestamp DESC`, taking the most recent row. Rationale: a re-upload supersedes the earlier file. + +- **Plan retrieval**: Use the existing `MagicPlanPostgresRepository.get_plan_by_uploaded_file_id` to fetch the parsed domain `Plan` from postgres. The lambda does not re-parse from S3 — that is the magic_plan lambda's responsibility. + +- **Error handling — two distinct cases**: + - No `uploaded_files` row found → raise with message indicating no MagicPlan has been uploaded for this deal. + - Row found but `get_plan_by_uploaded_file_id` returns `None` → raise with message indicating the plan has been uploaded but not yet parsed. + - Both use the same exception type; distinct messages enable diagnosis in CloudWatch. + +- **Spreadsheet generation**: + - Format: `.xlsx` via `openpyxl`. + - The template `.xlsx` is downloaded from S3 at the key given by env var `AUDIT_TEMPLATE_S3_KEY`. + - The template is loaded into memory (`openpyxl.load_workbook(BytesIO(template_bytes))`), populated, and serialised back to bytes for upload. + - Cell targeting uses named ranges defined in the template (`workbook.defined_names`). The initial stub implementation may use fixed cell addresses as a placeholder until the template is finalised. + - The template does not exist yet. For the initial implementation, the template download and population step is stubbed — the lambda generates a minimal valid `.xlsx` (e.g. one row per room with name and area) without a template. + +- **Output S3 key**: `documents/hubspot_deal_id/{hubspot_deal_id}/ventilation_audit.xlsx`. Re-running the lambda overwrites the previous file. + +- **Operation order**: S3 upload first, then `uploaded_files` DB insert. An orphaned S3 file on DB failure is harmless and will be overwritten on retry. A DB record pointing to a non-existent file is worse. + +- **New enum values** (added to `FileTypeEnum` and `FileSourceEnum`): + - `FileTypeEnum.VENTILATION_AUDIT = "ventilation_audit"` + - `FileSourceEnum.AUDIT_GENERATOR = "audit_generator"` + +- **New `UploadedFileRepository`**: A new repository (`UploadedFilePostgresRepository`) is introduced with at minimum a `get_latest_magic_plan_json_by_hubspot_deal_id(hubspot_deal_id: str) -> Optional[UploadedFile]` method. This queries the existing `uploaded_files` table via the existing SQLAlchemy model in `backend/app/db/models/uploaded_file.py`. Full DDD migration of the `UploadedFile` model to `infrastructure/postgres/` is out of scope for this PR. + +- **Idempotency**: No duplicate guard. The lambda always overwrites and always inserts a new `uploaded_files` row. The UI surfaces whether a record exists; the timestamp on the most recent row is authoritative. + +- **Environment variables**: + - `S3_BUCKET_NAME` (shared convention) + - `AUDIT_TEMPLATE_S3_KEY` (template location) + - `DATABASE_URL` (shared convention) + +- **Trigger**: The SQS message is sent by a UI action in a separate repo. No SQS publishing client is required in this PR. + +## Testing Decisions + +Good tests assert observable outputs given controlled inputs — they do not assert on internal call sequences or implementation details. Prefer mocking at the boundary of the system under test, not inside it. + +**Handler tests** (`tests/applications/audit_generator/test_audit_generator_handler.py`): +- Test that an invalid trigger body raises `ValidationError`. +- Test that the orchestrator is constructed with values derived from env vars and the trigger body. +- Test that the handler returns the expected value on success. +- Use `handler.__wrapped__` to bypass the `@subtask_handler` decorator (prior art: `test_magic_plan_handler.py`). + +**Orchestrator tests** (`tests/orchestration/audit_generator/test_audit_generator_orchestrator.py`): +- Mock `S3Client`, `UploadedFileRepository`, and `MagicPlanPostgresRepository` with `MagicMock(spec=...)`. +- Test happy path: correct S3 key used for output upload; `uploaded_files` insert called with correct `file_type` and `file_source`. +- Test error path: raises with appropriate message when no `uploaded_files` row found. +- Test error path: raises with appropriate message when plan not in postgres. +- Prior art: `tests/orchestration/magic_plan/test_magic_plan_orchestrator.py`. + +**Repository tests** (`tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py`): +- Integration tests against a real postgres `Engine` (prior art: `tests/repositories/magic_plan/test_magic_plan_postgres_repository.py`). +- Test that `get_latest_magic_plan_json_by_hubspot_deal_id` returns the most recent row by `s3_upload_timestamp` when multiple rows exist. +- Test that it returns `None` when no matching row exists. + +## Out of Scope + +- The SQS trigger — the UI button that sends the SQS message lives in a separate repo. +- The `.xlsx` template file itself — the template does not yet exist; the initial implementation stubs this step. +- Full DDD migration of the `UploadedFile` SQLAlchemy model from `backend/app/db/models/` to `infrastructure/postgres/` — this is a separate refactoring task. +- Named-range-based cell targeting — the stub uses fixed addresses or a minimal generated workbook; named ranges are the target interface once the template is designed. +- Any ventilation calculation or compliance logic — the spreadsheet is populated with raw plan data only. + +## Further Notes + +- The `audit-generator` application scaffold already exists at `applications/audit-generator/` with empty `handler.py` and `audit_generator_trigger_request.py` files. +- The `MagicPlanPostgresRepository.get_plan_by_uploaded_file_id` method (introduced in recent commits) is the correct entry point for fetching the parsed plan — no S3 re-parsing is needed. +- When the template is eventually created, it should define named ranges for every cell the lambda writes to. This decouples layout from code and means template redesigns require no code changes. +- The `openpyxl` library must be added to `applications/audit-generator/handler/requirements.txt`. From 41b282042ff26f4818b3d40b6b18bda943092cf9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:42:53 +0000 Subject: [PATCH 04/18] =?UTF-8?q?UploadedFile,=20FileTypeEnum,=20FileSourc?= =?UTF-8?q?eEnum=20importable=20from=20infrastructure.postgres.uploaded=5F?= =?UTF-8?q?file=5Ftable=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- CONTEXT.md | 10 +++ .../d1_ventilation_template.xlsx | Bin 0 -> 15658 bytes backlog/ventilation-audit-generator.md | 71 ++++++++++++------ tests/infrastructure/postgres/__init__.py | 0 .../postgres/test_uploaded_file_table.py | 17 +++++ 5 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 applications/audit-generator/d1_ventilation_template.xlsx create mode 100644 tests/infrastructure/postgres/__init__.py create mode 100644 tests/infrastructure/postgres/test_uploaded_file_table.py diff --git a/CONTEXT.md b/CONTEXT.md index 3580b93e..8234aa73 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -65,6 +65,16 @@ _Avoid_: user input, raw address, user_inputed_address The reference cohort matched to a target Property by both geographic proximity (postcode prefix / UPRN range) and physical similarity (property type, built form, age band); used by the EPC Prediction Service for gap-filling and anomaly detection. _Avoid_: neighbours, similar properties, peer set +### Survey documents + +**Ventilation Audit**: +A machine-generated `.xlsx` spreadsheet produced by the `audit-generator` Lambda from a property's parsed **MagicPlan Plan**. Written fields per room: room name, width, length, area. Per window: dimensions, opening type, number of openings, percent openable (`pct_openable`), trickle vent count and area per vent. Per door: width and undercut. Internal doors appear once per room they connect (so typically twice). Columns requiring human knowledge (Blocked, Pictured, FP reference numbers, door location labels) are left blank for the coordinator to complete. Recorded in `uploaded_files` with `file_type = VENTILATION_AUDIT` and `file_source = AUDIT_GENERATOR`. Distinct from a PAS 2023 Ventilation document, which is externally uploaded by a human. +_Avoid_: ventilation report, audit report, PAS ventilation (that is the external survey form) + +**PAS 2023 Ventilation**: +An externally-uploaded ventilation survey document produced by a human assessor and ingested from an external source (e.g. Coordination Hub). Recorded in `uploaded_files` with `file_type = PAS_2023_VENTILATION`. Distinct from a **Ventilation Audit**, which is machine-generated from MagicPlan floor plan data. +_Avoid_: ventilation audit (that is the generated output) + ### Source data **Site Notes**: diff --git a/applications/audit-generator/d1_ventilation_template.xlsx b/applications/audit-generator/d1_ventilation_template.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..057da026fc51d3b5e3abe2ca43732e8d8252c1d3 GIT binary patch literal 15658 zcmeHu1yogQw>Ah8(k)1Xgmi-vA`Q~r5}V${CZth9O1h;(>FzEm0coVW8>I7J;5o;0 z{Lc5?``vrT_{Sfc!Dg}EJ?Hz(=bdxDvF4VOf`P?`x_zl>b!qi8RC{B?K|$UAVgdIw zg{&;?K$dp8N>0`wTWv;1um!IG@J8wYg4mi`S@MyyFfx;qf-DWJ3{5SK`N)-(L>air zNp0=)EDiN6tSmu%@Dm_Nktq1ct4N;^5mqy z;l1sPk6h2%+QQU8&kg{}blb}G_kyp%M!yJ}f^U0vWVmhlCqnwBmcI!7xi<$(!(VJ- zure|-H2@h}8Q6mXLK&?rK(hK~AOpL9)&T@$Vf*j(@Ec={faIOcwkD?5wsil(a5HO= z@h@Hcu>7v!pBQd!`S&un_W==Ez%wAGh9FW>IX#G-j2;*m2PQ`gCR;lv3y|%t1O3wD zzek&`H3Xz*Xlnuj*?}z>e<=Ok#DCWHYb%KPorEpZ-5-|!VHAJp|Bry!0nQF$x_kfc zW4KfEFM9-hO%7sZZOa5u{NFYibQ@@fAVUUgKqUxbXA1hmwf>g(KRP$Cf`I=0-hcPy zTj~F7m$zPKq6YyPD%wGSWb{AoY=74MFNp_$-1>yRm6iFwZ}~5I;unX%b$mv!o~h-( z1n`r5ds{mz@P8YG!JV4F83(4jq|Pt*+a!W;H;Ftv#p0iXhJyM63k8J+Od`L~kskrJ znqWrnXbf0BA+y0%@I)Aa6B5cPliXZ0U*!JWf))lQwU~H4D@ZUHz~v_mu}EgRp1#@o zzEmuaxm|7FvPKvo=#BBM2m6H;D874xCgh{VH=|5 zq#M%OUmQTU0rzQzfcIl@fHU(pm0kIAnKjO;P8V}<8oQE`$rNV$G44BBspiwEqQr%X zO7!l>N2D{#X%0lv#hTUu6Wax-vGN?Gqc0wK@CvagSkz6K7_4i&7(~HOYkwg%Pk~`) zgO^R#rA+fUmg8%rJJcH2z_&AFz2kXU(oQ!@OdoGoVeRAv@9 zV`j`Nwjg0X#)b&ax_xr*jb`*n3f^lCFHJWSDx4pREHdWGVWMvY&^E^g*927=`WIYBN2!BTCZP5V)$v5h= zDcEV1SL#w@(*uW0e{}|!jHp8$U^S6w1_g!k!x?@~1>4%-ka%wNYdXJO^%l*0xs=MK zlj2d%xstU}J9N+yDKH5z#_@-CH&;DUoJF-%O&?wjufN#!p=Y%tZ1JfnOW?o&e26Ag=o>`ueXS|- zvf8MOu^G>%0INAv@2bx}@~_Ae-g7F{1-@DSERo(66S?n5em@#70N-b_sX&ObD6R3h zHVnlp(WkC_)nzSq@M#!N=ftC@Z;i#6-!axSZ&R+-B%#a%7)Tm1%DtL(pY~=h8RdWX zo>rs6I+Qw0;gx=D@9y>RJSjrF zxSH;sbgGn(3P}bRU#@F_vSlY$EAva~C=Th=whi7-)WYHGXrnCzqNqt4GS2<%`;nEg zXH6n6o(3(|hr~a23RGVLS6BrM_Ou~4Pj6x-M9GODz!e6TOttmw=b<1o+}NMd3gmzA zKT2{miLV*!5zXiGtoG-Oh!Owj0+oLrBkkm`z;gdhaGO?N3(TD3MM}YEobp-EtgK5^ zM*WVXlvq@~p^nsH40+u5zSrjZB&6&axxceOi!Lvkxv-c?TUxhw? zNw}MJR2<)2{LQL_wimZ2*kMMU;9VoM@Y7uvwHKea_iC>-87l}SjRO|Bj}4KiU>YJA zLhb6PHDkGn-+entdEcUrRo{w4(N=SCGL77?V zy+o*Z9b4XmHP5ehKT~qla+QSSxS2pW_b|ik1#P!>b5U%|@YigRhb1YlIs~E*aoqtBUDr8RX(Azsl{ug?}{e(Hz0}|psXNAoKPAO9S8^03$elZe66>Y zQ^BtpY&V``E|*i1u$&kj(Doc+cj*ihgOzLVSsnFgG8~MWnvhC+9M?-feB+F2$WkgZ z;=}O1PB&H_f<4xbPl_4Qjoeup$Rj{uegccSwq%aB(^0pMW9TmSGMs=aAuAO+gSIhL zyxmagMcN#$6LJplRp-dQw5G#ET%wA;7^q}R+d~6}q`AXY zaa5uEc^XOm8YMQ~i|`w8J~cXW{I`!7LS-EFwIUB-iu^y+p&t~3LLMNtF{jmu#*V!Z zBvct`e)8;tDc_r%MFV0oT87qfclr?QtS5NKSB_zQ zRky*}rL{W*I@zch+3B0EeXpjPbmfh|^?GW8D=8G_+Y5FABra}*yq@dX_ZzSEhR{$0B4$RUe@St?}oMv=vO1!(c`)7nn-brw>S&e1ajye=HqWXa413(I+$b zD&#zpCpGY}kFV9{G27F<(khR8-&L+`?I}8q58XBJH)@WZUs7Glp znX2mmH%c@yOT=h~X zb-ECrB_RsieX_XeOHaei`c(?19)*vOCSpI?a_fF2&V1dAhDaCb%AD1MqzHLqzU_8+ zsKAZa*XNOOlltaBVbjHvv(VoCI$A3>W(dvlRpkYP=z3vMNL}@5U(E7w-A##tXuG57 zdv=^P3Ps(S5W8(lCyz$fjxVP>x&g^zEd(*;lVcs_X@nMxmy0}BjJTL_rxF77mj0#8 zp120 zrnU66!>`wxIuEb!GYtmJqCa zwDvy3n)mGO(PFJa9G%(>(*<3Q>3rP~NjypfD9C;1!v+D09BN`$(7U4wpM0uaE-gr! zx6Hljc9}h^QgMM8O5@mk-ww%|_PT%*q@*Pt$;_3p%GjH&4jSKHRh#8q^}~QG-@0 zK!+t#WH-cvvw!uArcIkiOMpMr-GhQ6_~942w|nouEa7kW)VBHIf@ZDE=pu#WM!MZH zlC+N{IX{MiBM6_18+3n;_BI-9uPn4S!4i5@vnP4Uw9*S>TF!Hlc-kZk-g}Uf2K#mf zcJf@bS?}g}57|@ACAW4?zWEJo{Lu!{vtB1}tw$1u@2Kww^E8M{wCfdSBVf2SWD9L@ zti}@Hm-B4EEvL~Z8`glDB56O5J`vZ{d^?7QaRY;9KTY6@vND{$s_W3%7aF{T&Ld=R z7K(GJMA?!4gcWCrgfox_%-y#zV)#Dhpy4w_1`+Z!b=u|&nv80kgxj|j8M;i?<)gLC<&@6XyY5Zhw&gZhjDRN5+5g7>HNl&xaHL#nYg_9B=@qL4Vds zN-AN;Z2x&g|1nAbuuJ?qL_asA z{|poR|0JfrqW%f!mp$q2$raO|$ISMZJuP%4`xj1N_D}>O0RxDHKgRrbii5(n&0CnU z-`z-dxy7>wn3&KRwC19;uZ)dpi_4m;eHRQ@zdA9`9etp)#ah?r1oHqL99}8d4+-Q_ zygE$77w8@}(Rx(A5~s=tpEsFjbVd|ApIb(#0LPb=LT{hT?tkn4e!*-;Z{J7@oz*GX z>Z5zUHL>PQyq~e!sVJRa8Y!149s}&krf!+@-;Jr%K{IJ5uqH_?x z8ocs~6mC)=L+#;@2|t7uOJT17ZyhqeZ`#K{9z3caaNIn+@QYr4(p^AFj1TsmZb=s0 zf4qWqgSVZKbdnx(yp*n*^Jo`7{q159=<4uPLv^HeN5*o{{=>)a5z($>tPNtlhDKpep0Ao9mqJOa`d6AJC}`sLv(M)mo0_qFrFQOx&aTHK z|CJ+2F2o(OTYdmXygi}+>4<+s*2geuD=%i`=Qom(&!iDPx*}vffX?=nh#1`_7EBSR z6qNX)A$_qcRc=3mT5BQJef+&N5!|PL59ync607g^1ED5RZjC%Dk1IdB4)WXLmkt&R z;j4QsC{wuO=_+jB9G2j?->~!WYCOwCh&%4Q&qfy$%{WBI5Nn@G9J+B)ysT7C-TYAm zb`ix)w*AR`?=1IemlWQGs$u3tZlV*9mQe>?3DfEQ!M~ zjm*~so623dD~Od@j4GFP_t#SkapiiP-{)$Cp9$LX-)ti?WEyKivtSnB^@tDI>7(a| z`YWT&EE)>vz)=+zK<6!^Kx+8)pmsz}%65_)`I?S!y3+ZSj83sKqF{SmXmXouAmTZ7tpwkEwMEyniCeN@ zdwuPqbH!mB8XEgJHu`-}A|J-r4z%5UuO4wh^prTqCR^%=5@%~|46lcQuR$UY*9OD! z)LJ5~ro0!492v46InE3-Sbt4`>nm)IG5yeCDBnKm$()^N{h}Clg}j^2W=};MF7RuogR@C<9R3syX!xF{c2O9 z+J&lcP!JmlF`#-eo8(Ouya9*BoD!-d3vb?V$a3miDyFvwO>R0AM1e$~$=vcF!XA#+4NEWerg}!`rqNxG|-UPY9TgvI{nXhri5y3(x;gvL0e?Mz0PL)SY&)3dE2Z)Z=11w4_hjAyrn3ssqob!c)gmpF-G+9 z={40C%37WGapT%)qS^_u&pkuEDL&xIy=pgoQanS)wk9Um=D~QC!rvzl(%|wzic(sF z9-eX<+peIY?4pc|;9CR)i9NZ*(0A0XyU_q)neuQGSpq^fX(s`(j8wog{w#Io!C0D! zteUZ%FSh%&K=YLbsH$43wmm0f)I5I~WFaokKbSfhfCQPCaiQaE5nH#hS~fk7szJHewjJ z)o6O)OZLV`zGE6z&KDXqD-m$8gyZa?>lcr<({P~N->NFH`SB#>tD|X?_NoQnnbqNN z3mQpcKJ`)g$f|V1vF+fi{Pl%Q*_LwIb?a~vcj&(+kbRS7e|lh7SwI2>h4~K&z9-S3kXM7}0kIa$sXof5=brVa~M)$Gkb$1IEUo$#r4sC2F$ z#T@=Bq$$$hyd0!ri$5WmN}S#-h#~h{*788H8-a6vOBBBZrL$t(>0NO{kFzV=F?NqnCL!-4+Tgw9y%otYSq8a zqNrNJ^4U=wZY#?Z7kaADR`qDumo`3%+2>uSviU0U21EtZFrIKFNRLcUWg)gm3tYIE z-Qte#vc)!ZimRf7IDZvq4Z$MADf5(iD0amXRs!oF)&m)~(nqf%JeiCmjj4 zFxALb2i+OBfj4qf-Bg533_GF5MYiWPz2D$B{Tm^D0;%t}bMYxE4RdZV4<4e;H8cdj zPcz`|k|9ck7r>f`y6;lta6gzaY;FT9r_K${7rn{|bgdO99N6>tLAAG{9D4)#EA~8$ zJx7K8H;F`bix~bRs3aW*K{+fBRiD^C>L^&73uGslSjEp`XDm#VWO$dDB-N|a+{Vum zlF7NRmuA?py86Pi|gw*gC+d_`7>}cDH(q7G= z^kN6b%q_%-m$o?g7epK)RE)f!Lrw5ACcS(PFCP_b>ZT6l~nU;1c-1Xgy958+Tm>4^M;R7Uy|LFE7 z#@P>S`FVjl>Dm`J15z^}xWIa0Dy)SpA9q$uBeDgTclY+Q$@OO+k)&e82k;@|b1pUz zDk zy|TU=Ghe>s`QscT7bbWj1A<4npZ1pwN7;fLn&~H~6FxgyOZcUMASKB z{LWCW6c@``daU3WzV7BJ%N}gzrZr^VtiwHq1cf| z%o4ZzV!YT0X+wP)ui6&)HvY$_YGgIgQfn0-oxtVvH~ulmlY zxGxl1ilg0g9v`=57!(gEvN$U9S(Bc#-?-P#yq#<-{62ktdU93SHf`n= zGSlFm;KsC|?{-#2dtp#tMdNXkZgEh!?d=tRcHSKg4px*AaNaPw5MpI=&o>hw$-@f~ zv?+C0otm!axlDfT>bTdZZ$qNn%lYgnqvkX(K_dgM2iVA?ZboVPa8Hi#l1L!B%gI(P zY1mj~CQ7@fVDx)(TbsrAEJpas$qD|^Z;z|g8Lw%HIeYU1Y8TADpXT6hJxJ(mo!G54 zAkMqCQlGMD|3q!pSSYP{l-xTiQ)$E0*IzlF<5*cGwfb#ylFN7bYNe(s?y5>tEKrvu zV8bvs=7p7{P87A%#jaN_N$$}?gNBmjs{3`(h3)alai8u;r_|%a{V^AfnFcu}q_&|K zEbcs`8bR$H$5AC3v?p}B#4d$Y2fCx#W{DFC-)H0L)XklTbG-{B*WUUPD{qR*ZaVKA zRcScZ3A}RBYo8cO_uy$M;9!r7x3>~QmX80c^ZB>tYw?=qr za-NM{zKRaMVBo0UJ8s?+Zlv3avv9nycGPKX{b1+V5#8%fr?NS8wsSqM(=H79?j;eo z7QRpT=ZBBsc=>!OG0bTWzNe&v95j^ z{7DRa6Vqw0nbM3l$0xsaw-tIVe-c)I{k`>uQ#LVPb6fuY^%(RS>nBDdqByAEbqs5GJCN#>R7#m3P< zpR{3*{cP`^&Nh>)NZkba?pc~Z@md*qT7@{T?qS0Rg+TaDG4w|Aa_=%iNqzyPH>_lx z_z#jRu~qtzbW?haLlG$3QyY{FlgGQGy*5|jW#zcNlCdn7q5mdMJ$s&m`>C^jU*|UCD(1Gihj%Zq)MV2{L+qxZh z0cFLFm=1H^C!S`(d3~n52rU&31nYdcD_Q3WqUe3OJT?58VKISbU$gi#iwpZk5*E)B z+I{>cR@e)KwR*F4A%!-yjJZ6~=1E(hI?vZJf^wrOdQG(g^1w2S3QHpi=gN3NQv1pd zd`e13a`K}W#b*2}4$pUz{bZjgxM<{eUPE6rkYCic!H0&Bx{Dn5IVa2UH07W#<&j@> zobYUDy&Ac-Ueq@RXFRz$gFNC&8jTXB>$UK9a!Nx^0bO=39W9uPZ@Mn@SxViNPhRKs zVA95l^mX2c(#EmXSX=qt@#Lw#Sgg07Nn=P6#UF&fT_V-B|CUX-Ewj9$_syPMtQV^q z#!UA*gZx4%LB%k+=>%u|+nHRL#AIMXNQIyWR~Faq+?0jY{mN31jhkbUn{gY7^wq(; z@()(C3unyE`^9x$Gi8xy-c&H`_$s~=nJanZX-9L$>FetDdk4tp>3Dh_&3(D7N14O} zuGBXF7h-|VLQ(AB|lT8C=LLoL_U#^2_{`OM3sZKcs z+U%!vJaw^Wx?zWynKchBNBTuzF;$nUw)@kL`eVi{JxgfC z;lfo7ryNYio!1IGwACNoxQw><<^_e7vy^a(j<$DBj74)yrU(!xaj;{jq*_~aPdGZ> z-hQYqgoh!{3*5Kh5*)w9-qBjhx$|XWX||Np)ClaECmmXjSONqNRYl%7+*Q|JUTP{7 ziVv3NC>8KQ;C3fLjo_nD1Cz}Va`*?Im^8% z8goySWD^9p!%BnCjmoY>GasEU%KT1SbSldnefcG)4 zzGNiEG91~L+895OZuNSuMLqpF`R2x^xpFSUqE5(rm(Hq=_bxp@-i&*zz{6bW`CKX^ z?gk)L=-LM;8=WP_)zurv@ww`XeR&-{X;FSOoLNVx&&(QIA+v~dnZo$;mU5fm_t|J-1@(GUZgqOG`W4)Bz#$k86PynfY({=ygq|-9zy|KS zQRgLKf`qGq>oHmk-f;@APy`&OXu&RINh3K{ufY$3lUY`x@E~;U`&&=gvnad<$-Khx zBYQ|S{QdO3;6UgXGHN`JzxY9b>!R<|PO?Xxq%#SHou$Z6jO19n0`Qeaktamm*q+jO ztvz|Ankq0oPlH)H-Ny7n)NX zm1u~G0>u0m2GSf=g7k@w_~OOazmQ5lsC~efE46RI`jO77jF#B`fhNF3pOI1xtN6zs zrnqqC4@Nc16fPX0(-HRsh?Or4^$QU@fv z7vxKefbl)H_ynj`V)!vE#zPF+03Ze(wE2bRDn}(o;>19)^@V{NN0se6K#l;QUZh(< zF8dY-w}6<4kC1)tlt^R**Rklv_>E}h6Ca}uz(ZF+PV^7Mqfh$o3fklUDOX^!xdktKI_ z#B4@W-0HNsD#gS(FvHIa%>os@I11rc5-&II=iG>wza(+2Ku*hjqwND1G{L48RAH1t z=QZ-dhQSNx!!YaJ=xJeGCV!|v*8e>of_G86US(2a@bUV;q0R;z@y?L}pZy0%j6?+- zImPhK5##Ate|lp4BHEWfQ~-8)CJFaPAUI>{9Z`vB6{I^Y!L%Bk`B1E$`!-3}Zs zYLfTw1Oavn{|@0!{&QaLEIgf3Bbo-Ezx^;C>Q99L6@X?z*%rVP)d1By&-9ifX(+z< zcu`{{CJ7(zQQrIQ!-BHpYZBfg5o%&?fc~-DU;qI9UoL;VM~RsveDM7V+z2bp{?|(v z30Zx>JM+^U@QK0&Z^xlO@M zQHCc|haFOeCkYQmi04ul2cHi%M-<(<*WiXkhS!fa@7vsHw?{ekTV9famnDD3=f5>C zLCNdCv?_s|V;l;ox#aZ+l3AI1(&zW6Uq}UHTkdkx+MW|;>8B(?_oO>DKw9nWi}>O6 za4271o#;xAv_^zLoiT420qnH?EkYQHc>RA8 za`%}!wi_J)3P1u7zkum?@muDMxjrXB$>*Q(BYuf~#xJ0)6Nq0IM@&Rn^7S8|0eJm? zOg;_BU-xCx)&P$H$;~I8odhI(2Aokt44CdHz3w;^+{R~57h*LT9cDF=WAL(gOdfzj z5H9$8kl$X9tV<}N$)?RoVDtH>QOh4v1|$iq-)0F_AfepOx=JdtX@tB#+HMs@c~Q^* zHeG4)IAZ>9m%qw?2;|y@82$bT8fQ$ATS6l5vefc#SqgMS`K9?s+g+9eK>tnvV2eDE z<&O3!A0!D6{D1lXB7Y})lmtn_LlF;^x>MSpec$Ea^9MYy3|QRSiNlTGEt@9TOhkPR z`}49rQpx!c|87Qq?Tk6y8e!syDJ`3}F2T;{|A~4)MHX0PlzUO@-L6N=Mu9{tmd z4{%|ZNj>gv{u-xT(l<+E^hue3)vlk6-!?_`PAh~f*ScA3&5Y2f_t&~{aGrLBBH1O5V5(Q`yV*~o;%5@zcdfY| zsqN`FI!|Ab2K5jJ$Me0eJEF+#%c=?^vojj8otjQfUG{kwWBQsmMj|MqU4f2Q_ z3IR{WC^iOpgb&Grc|(U%f_xza75i*FeCCU*5a*e8rlrHQAdl#wJ}__iP*zX}grIU? zil^RU(LaF-%o{aS7GwKz{e))T!oo)KWRO+#w$Hln~kB3^aK*4ID z??^M4r8Y^xRTtb~b~$$eaVP2_2~a+2TMQ}P72(!&PmaOb4TwJ=JJ?t@Ik-$d5UrZ3 zZ#N^-WFqFG=ve-~T-jFRDq`e5Nm!+3>5dhoyiBxgRIK(IMa8YH?Yv{Ld2y>)t^K=4 zz;;YyVS&->ww2uK?)s^oZ@Ream(Fgr^)413(~ShYW0Bmtmd~nmE}c&yn!E`&^wGZX zfy;G{YJHr>tG)A!`qesLS2kO{NOBkSZD#_OC2ZI(Y$|7_BO9iB3$D*+TkV~n>Ecb= zI@TwEWtGkzl<`jQ;+~vw_vMqOx(t@i}LAWop&H4yWRrhEliJlCqVn1$8Syx*iPC9$+gE zs~ZwIDQKX$b7}7aH)y z_)le(KLvkK_iNqRtFt023&!;n3;+NC literal 0 HcmV?d00001 diff --git a/backlog/ventilation-audit-generator.md b/backlog/ventilation-audit-generator.md index 6af37101..242c6569 100644 --- a/backlog/ventilation-audit-generator.md +++ b/backlog/ventilation-audit-generator.md @@ -18,9 +18,7 @@ An AWS Lambda (`audit-generator`) triggered via SQS receives a HubSpot deal ID, 6. As an engineer, I want the lambda to raise a clear error if no MagicPlan JSON has been uploaded for the deal, so that misconfigured triggers are diagnosed quickly. 7. As an engineer, I want the lambda to raise a distinct error if a MagicPlan JSON exists but has not yet been parsed into the database, so that timing issues are distinguishable from missing data. 8. As an engineer, I want the generated spreadsheet recorded in `uploaded_files` with a `VENTILATION_AUDIT` file type, so that the UI and other systems can query for its existence. -9. As an engineer, I want the audit template to be resolved from an environment variable, so that different templates can be used in staging and production without a code deploy. -10. As an engineer, I want the lambda to follow the `@subtask_handler()` pattern, so that it integrates with the task orchestration system and benefits from standard error handling and observability. -11. As an engineer, I want the spreadsheet cells to be written via named ranges defined in the template, so that template layout changes do not require code changes. +9. As an engineer, I want the lambda to follow the `@subtask_handler()` pattern, so that it integrates with the task orchestration system and benefits from standard error handling and observability. ## Implementation Decisions @@ -37,10 +35,11 @@ An AWS Lambda (`audit-generator`) triggered via SQS receives a HubSpot deal ID, - **Spreadsheet generation**: - Format: `.xlsx` via `openpyxl`. - - The template `.xlsx` is downloaded from S3 at the key given by env var `AUDIT_TEMPLATE_S3_KEY`. - - The template is loaded into memory (`openpyxl.load_workbook(BytesIO(template_bytes))`), populated, and serialised back to bytes for upload. - - Cell targeting uses named ranges defined in the template (`workbook.defined_names`). The initial stub implementation may use fixed cell addresses as a placeholder until the template is finalised. - - The template does not exist yet. For the initial implementation, the template download and population step is stubbed — the lambda generates a minimal valid `.xlsx` (e.g. one row per room with name and area) without a template. + - The template `d1_ventilation_template.xlsx` is bundled with the lambda at `applications/audit-generator/d1_ventilation_template.xlsx` and loaded from the deployment package via `importlib.resources` or a path relative to the handler file. No S3 round-trip for the template. + - The template is loaded with `openpyxl.load_workbook(path)` (default `data_only=False` to preserve formulas), populated, and serialised to bytes via `BytesIO` for upload. + - Cell targeting uses fixed column letters (see Spreadsheet Layout below). Named ranges are not defined in the template. + - The template has formulas in columns J (`=H*I`), N (`=J*M`), S (`=Q*R`), and Y (`=W*X`) — the lambda does not write to these cells; they are calculated by Excel/Sheets when the file is opened. + - The template has 50 data rows (rows 6–55), extended programmatically. The footer merge sits at A56:Z56; legend rows at 57–60. - **Output S3 key**: `documents/hubspot_deal_id/{hubspot_deal_id}/ventilation_audit.xlsx`. Re-running the lambda overwrites the previous file. @@ -50,13 +49,16 @@ An AWS Lambda (`audit-generator`) triggered via SQS receives a HubSpot deal ID, - `FileTypeEnum.VENTILATION_AUDIT = "ventilation_audit"` - `FileSourceEnum.AUDIT_GENERATOR = "audit_generator"` -- **New `UploadedFileRepository`**: A new repository (`UploadedFilePostgresRepository`) is introduced with at minimum a `get_latest_magic_plan_json_by_hubspot_deal_id(hubspot_deal_id: str) -> Optional[UploadedFile]` method. This queries the existing `uploaded_files` table via the existing SQLAlchemy model in `backend/app/db/models/uploaded_file.py`. Full DDD migration of the `UploadedFile` model to `infrastructure/postgres/` is out of scope for this PR. +- **DDD migration of `UploadedFile`**: The existing `backend/app/db/models/uploaded_file.py` (SQLAlchemy `Base`) is replaced by `infrastructure/postgres/uploaded_file_table.py` (SQLModel). `FileTypeEnum`, `FileSourceEnum`, and `UploadedFile` all move there. The class name `UploadedFile` is kept (no `Model` suffix — there is no domain counterpart). All seven consumers update their import path; `backend/app/db/models/uploaded_file.py` is deleted. Because `UploadedFile` is now registered on `SQLModel.metadata`, the shared `tests/conftest.py` `db_engine` fixture must emit `CREATE TYPE IF NOT EXISTS` for `file_type` and `file_source` via raw SQL before calling `SQLModel.metadata.create_all(engine)` — otherwise the table creation fails for all integration tests. The dedicated per-test conftest approach (Question 6) is therefore superseded. -- **Idempotency**: No duplicate guard. The lambda always overwrites and always inserts a new `uploaded_files` row. The UI surfaces whether a record exists; the timestamp on the most recent row is authoritative. +- **New `UploadedFileRepository`**: A new repository (`UploadedFilePostgresRepository`) is introduced with a `get_latest_by_hubspot_deal_id(hubspot_deal_id: str, file_type: FileTypeEnum) -> Optional[UploadedFile]` method. Queries `uploaded_files` filtered by `hubspot_deal_id` and `file_type`, ordered by `s3_upload_timestamp DESC`, returning the most recent row. + +- **Session management**: A dedicated `AuditGeneratorUnitOfWork` context manager (standalone — does not inherit from `PostgresUnitOfWork` or `UnitOfWork`) holds `uploaded_file: UploadedFilePostgresRepository` and `magic_plan: MagicPlanPostgresRepository`, both bound to the same session. Opens the session on `__enter__`, rolls back and closes on `__exit__`, exposes `commit()`. The handler holds a module-scoped engine (reused across warm Lambda invocations) and passes a `session_factory` callable to `AuditGeneratorUnitOfWork` — the session is created fresh per invocation and never long-lived. + +- **Idempotency**: No duplicate guard. `uploaded_files` is append-only — the lambda always inserts a new row; rows are never updated or deleted. The S3 file is always overwritten at the fixed key. The UI and any future queries treat the most recent row by `s3_upload_timestamp` as authoritative. - **Environment variables**: - `S3_BUCKET_NAME` (shared convention) - - `AUDIT_TEMPLATE_S3_KEY` (template location) - `DATABASE_URL` (shared convention) - **Trigger**: The SQS message is sent by a UI action in a separate repo. No SQS publishing client is required in this PR. @@ -72,28 +74,53 @@ Good tests assert observable outputs given controlled inputs — they do not ass - Use `handler.__wrapped__` to bypass the `@subtask_handler` decorator (prior art: `test_magic_plan_handler.py`). **Orchestrator tests** (`tests/orchestration/audit_generator/test_audit_generator_orchestrator.py`): -- Mock `S3Client`, `UploadedFileRepository`, and `MagicPlanPostgresRepository` with `MagicMock(spec=...)`. -- Test happy path: correct S3 key used for output upload; `uploaded_files` insert called with correct `file_type` and `file_source`. -- Test error path: raises with appropriate message when no `uploaded_files` row found. -- Test error path: raises with appropriate message when plan not in postgres. -- Prior art: `tests/orchestration/magic_plan/test_magic_plan_orchestrator.py`. +- Mock `S3Client` with `MagicMock(spec=S3Client)`. Mock the `AuditGeneratorUnitOfWork` factory: the factory returns a mock UoW whose `__enter__` returns itself and whose `.uploaded_file` and `.magic_plan` attributes are mock repos. +- Test happy path: correct S3 key used for output upload; `uploaded_files` insert called with correct `file_type` and `file_source`; `uow.commit()` called. +- Test error path: raises with appropriate message when `uploaded_file_repo.get_latest_by_hubspot_deal_id` returns `None`. +- Test error path: raises with appropriate message when `magic_plan_repo.get_plan_by_uploaded_file_id` returns `None`. **Repository tests** (`tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py`): -- Integration tests against a real postgres `Engine` (prior art: `tests/repositories/magic_plan/test_magic_plan_postgres_repository.py`). -- Test that `get_latest_magic_plan_json_by_hubspot_deal_id` returns the most recent row by `s3_upload_timestamp` when multiple rows exist. +- Integration tests using the shared `db_engine` fixture. The fixture already calls `SQLModel.metadata.create_all(engine)`; after the DDD migration `UploadedFile` is in `SQLModel.metadata`, so no dedicated conftest is needed. The shared `tests/conftest.py` must emit `CREATE TYPE IF NOT EXISTS` for `file_type` and `file_source` before `create_all`. +- Test that `get_latest_by_hubspot_deal_id` returns the most recent row by `s3_upload_timestamp` when multiple rows with the same `file_type` exist. - Test that it returns `None` when no matching row exists. +- Test that it filters correctly by `file_type` (a row with a different `file_type` is not returned). ## Out of Scope - The SQS trigger — the UI button that sends the SQS message lives in a separate repo. -- The `.xlsx` template file itself — the template does not yet exist; the initial implementation stubs this step. -- Full DDD migration of the `UploadedFile` SQLAlchemy model from `backend/app/db/models/` to `infrastructure/postgres/` — this is a separate refactoring task. -- Named-range-based cell targeting — the stub uses fixed addresses or a minimal generated workbook; named ranges are the target interface once the template is designed. - Any ventilation calculation or compliance logic — the spreadsheet is populated with raw plan data only. +## Spreadsheet Layout + +Sheet name: `D1 Ventilation`. Data starts at row 6. The three series run in parallel columns — each row may contain room data, window data, and door data independently; the longest series determines the last row used. + +| Column | Content | Source | +|--------|---------|--------| +| B | Room name | `Room.name` | +| D | Room area (m²) | `Room.area_m2` | +| G | Window location (room name) | `Room.name` (parent room) | +| H | Window width (m) | `Window.width_m` | +| I | Window height (m) | `Window.height_m` | +| J | Window area (m²) | **formula** `=H*I` — do not write | +| K | Opening type | `WindowVentilation.opening_type` | +| L | Number of openings | `WindowVentilation.num_openings` | +| M | % of window (decimal) | `WindowVentilation.pct_openable / 100` | +| N | Total opening area (m²) | **formula** `=J*M` — do not write | +| O | Blocked | leave blank (visual check by auditor) | +| P | Pictured | leave blank (visual check by auditor) | +| Q | Trickle vent effective area per vent (mm²) | `WindowVentilation.trickle_vent_area_mm2` | +| R | Number of trickle vents | `WindowVentilation.num_trickle_vents` | +| S | Total trickle vent area (mm²) | **formula** `=Q*R` — do not write | +| V | Door location (room name) | `Room.name` (parent room) | +| W | Door width (mm) | `Door.width_mm` | +| X | Door undercut (mm) | `DoorVentilation.undercut_mm` | +| Y | Door area (mm²) | **formula** `=W*X` — do not write | + +Internal doors appear once per room they connect (typically twice). `WindowVentilation` and `DoorVentilation` fields are `Optional`; write `0` when `None` so formula cells (J, N, S, Y) do not produce `#VALUE!` errors. + ## Further Notes - The `audit-generator` application scaffold already exists at `applications/audit-generator/` with empty `handler.py` and `audit_generator_trigger_request.py` files. -- The `MagicPlanPostgresRepository.get_plan_by_uploaded_file_id` method (introduced in recent commits) is the correct entry point for fetching the parsed plan — no S3 re-parsing is needed. -- When the template is eventually created, it should define named ranges for every cell the lambda writes to. This decouples layout from code and means template redesigns require no code changes. +- The `MagicPlanPostgresRepository.get_plan_by_uploaded_file_id` method is the correct entry point for fetching the parsed plan — no S3 re-parsing is needed. - The `openpyxl` library must be added to `applications/audit-generator/handler/requirements.txt`. +- The template (`d1_ventilation_template.xlsx`) has 50 data rows (rows 6–55) with formulas in columns J, N, S, Y. If a property exceeds 50 windows, rooms, or doors the lambda should raise a clear error rather than silently truncating. diff --git a/tests/infrastructure/postgres/__init__.py b/tests/infrastructure/postgres/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/infrastructure/postgres/test_uploaded_file_table.py b/tests/infrastructure/postgres/test_uploaded_file_table.py new file mode 100644 index 00000000..86bee933 --- /dev/null +++ b/tests/infrastructure/postgres/test_uploaded_file_table.py @@ -0,0 +1,17 @@ +from infrastructure.postgres.uploaded_file_table import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) + + +def test_file_type_enum_has_ventilation_audit() -> None: + assert FileTypeEnum.VENTILATION_AUDIT.value == "ventilation_audit" + + +def test_file_source_enum_has_audit_generator() -> None: + assert FileSourceEnum.AUDIT_GENERATOR.value == "audit_generator" + + +def test_uploaded_file_is_importable() -> None: + assert UploadedFile.__tablename__ == "uploaded_files" From 5178cd02c59b695fc4d1402c97886cd1c8fc0b94 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:50:51 +0000 Subject: [PATCH 05/18] =?UTF-8?q?UploadedFile,=20FileTypeEnum,=20FileSourc?= =?UTF-8?q?eEnum=20importable=20from=20infrastructure.postgres.uploaded=5F?= =?UTF-8?q?file=5Ftable=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../db/functions/uploaded_files_functions.py | 2 +- backend/app/db/models/uploaded_file.py | 69 ------------- backend/ecmk_fetcher/ecmk_service.py | 2 +- backend/ecmk_fetcher/reports.py | 2 +- .../ecmk_fetcher/tests/test_ecmk_service.py | 2 +- backend/ecmk_fetcher/tests/test_upload.py | 2 +- backend/ecmk_fetcher/upload.py | 2 +- backend/pashub_fetcher/core_files.py | 2 +- backend/pashub_fetcher/pashub_service.py | 2 +- .../tests/test_pashub_service.py | 2 +- .../postgres/uploaded_file_table.py | 96 +++++++++++++++++++ orchestration/magic_plan_orchestrator.py | 2 +- tests/conftest.py | 40 +++++++- .../test_magic_plan_orchestrator.py | 2 +- 14 files changed, 146 insertions(+), 81 deletions(-) delete mode 100644 backend/app/db/models/uploaded_file.py create mode 100644 infrastructure/postgres/uploaded_file_table.py diff --git a/backend/app/db/functions/uploaded_files_functions.py b/backend/app/db/functions/uploaded_files_functions.py index 3708813a..44af55ea 100644 --- a/backend/app/db/functions/uploaded_files_functions.py +++ b/backend/app/db/functions/uploaded_files_functions.py @@ -3,7 +3,7 @@ from typing import Optional from sqlalchemy import select from backend.app.db.connection import db_read_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/app/db/models/uploaded_file.py b/backend/app/db/models/uploaded_file.py deleted file mode 100644 index e00acbe1..00000000 --- a/backend/app/db/models/uploaded_file.py +++ /dev/null @@ -1,69 +0,0 @@ -import enum -from sqlalchemy import TIMESTAMP, BigInteger, Column, Text, Enum as SqlEnum - -from backend.app.db.base import Base - - -class FileTypeEnum(enum.Enum): - PHOTO_PACK = "photo_pack" - SITE_NOTE = "site_note" - RD_SAP_SITE_NOTE = "rd_sap_site_note" - PAS_2023_VENTILATION = "pas_2023_ventilation" - PAS_2023_CONDITION = "pas_2023_condition" - PAS_SIGNIFICANCE = "pas_significance" - PAR_PHOTO_PACK = "par_photo_pack" - PAS_2023_PROPERTY = "pas_2023_property" - PAS_2023_OCCUPANCY = "pas_2023_occupancy" - ECMK_SITE_NOTE = "ecmk_site_note" - ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" - ECMK_SURVEY_XML = "ecmk_survey_xml" - MAGIC_PLAN_JSON = "magic_plan_json" - IMPROVEMENT_OPTION_EVALUATION = "improvement_option_evaluation" - MEDIUM_TERM_IMPROVEMENT_PLAN = "medium_term_improvement_plan" - RETROFIT_DESIGN_DOC = "retrofit_design_doc" - MCS_COMPLIANCE_CERTIFICATE = "mcs_compliance_certificate" - OTHER = "other" - - -class FileSourceEnum(enum.Enum): - PAS_HUB = "pas hub" - COORDINATION_HUB = "coordination_hub" - SHAREPOINT = "sharepoint" - HUBSPOT = "hubspot" - ECMK = "ecmk" - MAGIC_PLAN = "magic_plan" - - -class UploadedFile(Base): - __tablename__ = "uploaded_files" - - id = Column(BigInteger, primary_key=True, autoincrement=True) - - s3_file_bucket = Column(Text, nullable=False) - s3_file_key = Column(Text, nullable=False) - s3_upload_timestamp = Column(TIMESTAMP(timezone=True), nullable=False) - - landlord_property_id = Column(Text, nullable=True) - uprn = Column(BigInteger, nullable=True) - hubspot_listing_id = Column(BigInteger, nullable=True) - hubspot_deal_id = Column(Text, nullable=True) - - file_type = Column( - SqlEnum( - FileTypeEnum, - name="file_type", - create_type=False, - values_callable=lambda enum_cls: [e.value for e in enum_cls], - ), - nullable=True, - ) - - file_source = Column( - SqlEnum( - FileSourceEnum, - name="file_source", - create_type=False, - values_callable=lambda enum_cls: [e.value for e in enum_cls], - ), - nullable=True, - ) diff --git a/backend/ecmk_fetcher/ecmk_service.py b/backend/ecmk_fetcher/ecmk_service.py index 35b8f552..fa613e4b 100644 --- a/backend/ecmk_fetcher/ecmk_service.py +++ b/backend/ecmk_fetcher/ecmk_service.py @@ -7,7 +7,7 @@ from backend.app.db.connection import db_session from backend.app.db.functions.uploaded_files_functions import ( get_uploaded_file_by_listing_type_and_source, ) -from backend.app.db.models.uploaded_file import FileSourceEnum, FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum from backend.documents_parser.db_writer import save_epc_property_data from backend.documents_parser.parser import parse_site_notes_pdf from backend.ecmk_fetcher.address_list import ( diff --git a/backend/ecmk_fetcher/reports.py b/backend/ecmk_fetcher/reports.py index d2f8ea52..2d77ab45 100644 --- a/backend/ecmk_fetcher/reports.py +++ b/backend/ecmk_fetcher/reports.py @@ -1,6 +1,6 @@ from enum import Enum -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum class FileDownloadButtonType(Enum): diff --git a/backend/ecmk_fetcher/tests/test_ecmk_service.py b/backend/ecmk_fetcher/tests/test_ecmk_service.py index 703bc4c5..b8c8cc53 100644 --- a/backend/ecmk_fetcher/tests/test_ecmk_service.py +++ b/backend/ecmk_fetcher/tests/test_ecmk_service.py @@ -1,7 +1,7 @@ from typing import Dict from unittest.mock import MagicMock, call, patch -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum from backend.ecmk_fetcher.address_list import PropertyRow from backend.ecmk_fetcher.ecmk_service import EcmkService from backend.ecmk_fetcher.reports import FileDownloadButtonType diff --git a/backend/ecmk_fetcher/tests/test_upload.py b/backend/ecmk_fetcher/tests/test_upload.py index 79823e8e..7cb6e8a9 100644 --- a/backend/ecmk_fetcher/tests/test_upload.py +++ b/backend/ecmk_fetcher/tests/test_upload.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, call, patch import pytest -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum from backend.ecmk_fetcher.upload import upload_file_to_s3_and_record diff --git a/backend/ecmk_fetcher/upload.py b/backend/ecmk_fetcher/upload.py index fc05363c..bf13a36b 100644 --- a/backend/ecmk_fetcher/upload.py +++ b/backend/ecmk_fetcher/upload.py @@ -3,7 +3,7 @@ import os from typing import cast from backend.app.db.connection import db_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/pashub_fetcher/core_files.py b/backend/pashub_fetcher/core_files.py index c387e0b8..c6392e86 100644 --- a/backend/pashub_fetcher/core_files.py +++ b/backend/pashub_fetcher/core_files.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Optional -from backend.app.db.models.uploaded_file import FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileTypeEnum class CoreFiles(Enum): diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 86a553f0..64c2f6ae 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from typing import Callable, List, NamedTuple, Optional, cast from backend.app.db.connection import db_session -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index ccb80ac4..09635c92 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -4,7 +4,7 @@ from typing import Any, Callable, Optional from unittest.mock import MagicMock, call, patch -from backend.app.db.models.uploaded_file import FileSourceEnum, FileTypeEnum +from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, diff --git a/infrastructure/postgres/uploaded_file_table.py b/infrastructure/postgres/uploaded_file_table.py new file mode 100644 index 00000000..8f1a7e64 --- /dev/null +++ b/infrastructure/postgres/uploaded_file_table.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import enum +from typing import ClassVar, Optional + +from sqlalchemy import TIMESTAMP, BigInteger, Column, Text +from sqlalchemy import Enum as SqlEnum +from sqlmodel import Field, SQLModel + + +class FileTypeEnum(enum.Enum): + PHOTO_PACK = "photo_pack" + SITE_NOTE = "site_note" + RD_SAP_SITE_NOTE = "rd_sap_site_note" + PAS_2023_VENTILATION = "pas_2023_ventilation" + PAS_2023_CONDITION = "pas_2023_condition" + PAS_SIGNIFICANCE = "pas_significance" + PAR_PHOTO_PACK = "par_photo_pack" + PAS_2023_PROPERTY = "pas_2023_property" + PAS_2023_OCCUPANCY = "pas_2023_occupancy" + ECMK_SITE_NOTE = "ecmk_site_note" + ECMK_RD_SAP_SITE_NOTE = "ecmk_rd_sap_site_note" + ECMK_SURVEY_XML = "ecmk_survey_xml" + MAGIC_PLAN_JSON = "magic_plan_json" + IMPROVEMENT_OPTION_EVALUATION = "improvement_option_evaluation" + MEDIUM_TERM_IMPROVEMENT_PLAN = "medium_term_improvement_plan" + RETROFIT_DESIGN_DOC = "retrofit_design_doc" + MCS_COMPLIANCE_CERTIFICATE = "mcs_compliance_certificate" + OTHER = "other" + VENTILATION_AUDIT = "ventilation_audit" + + +class FileSourceEnum(enum.Enum): + PAS_HUB = "pas hub" + COORDINATION_HUB = "coordination_hub" + SHAREPOINT = "sharepoint" + HUBSPOT = "hubspot" + ECMK = "ecmk" + MAGIC_PLAN = "magic_plan" + AUDIT_GENERATOR = "audit_generator" + + +def _enum_values(enum_cls: type[enum.Enum]) -> list[str]: + return [e.value for e in enum_cls] + + +class UploadedFile(SQLModel, table=True): + __tablename__: ClassVar[str] = "uploaded_files" # pyright: ignore[reportIncompatibleVariableOverride] + + id: Optional[int] = Field( + default=None, + sa_column=Column(BigInteger, primary_key=True, autoincrement=True), + ) + s3_file_bucket: str = Field(sa_column=Column(Text, nullable=False)) + s3_file_key: str = Field(sa_column=Column(Text, nullable=False)) + s3_upload_timestamp: object = Field( + sa_column=Column(TIMESTAMP(timezone=True), nullable=False) + ) + + landlord_property_id: Optional[str] = Field( + default=None, sa_column=Column(Text, nullable=True) + ) + uprn: Optional[int] = Field( + default=None, sa_column=Column(BigInteger, nullable=True) + ) + hubspot_listing_id: Optional[int] = Field( + default=None, sa_column=Column(BigInteger, nullable=True) + ) + hubspot_deal_id: Optional[str] = Field( + default=None, sa_column=Column(Text, nullable=True) + ) + + file_type: Optional[str] = Field( + default=None, + sa_column=Column( + SqlEnum( + FileTypeEnum, + name="file_type", + create_type=False, + values_callable=_enum_values, + ), + nullable=True, + ), + ) + file_source: Optional[str] = Field( + default=None, + sa_column=Column( + SqlEnum( + FileSourceEnum, + name="file_source", + create_type=False, + values_callable=_enum_values, + ), + nullable=True, + ), + ) diff --git a/orchestration/magic_plan_orchestrator.py b/orchestration/magic_plan_orchestrator.py index a3191985..9cbd9129 100644 --- a/orchestration/magic_plan_orchestrator.py +++ b/orchestration/magic_plan_orchestrator.py @@ -8,7 +8,7 @@ from domain.magicplan.api.response import MagicPlanPlan, PlanSummary from domain.magicplan.mapper import map_plan from domain.magicplan.models import Plan -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, diff --git a/tests/conftest.py b/tests/conftest.py index 0a246372..17c2eefd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,12 +17,13 @@ from typing import Any import pytest from psycopg import Connection from pytest_postgresql import factories -from sqlalchemy import Engine +from sqlalchemy import Engine, text from sqlmodel import SQLModel, create_engine # Importing the SQLModel row modules registers their tables on # SQLModel.metadata so ``create_all`` builds the full schema. Imports look # unused; they aren't. +import infrastructure.postgres.uploaded_file_table as _uf_table # pyright: ignore[reportUnusedImport] # pg_ctl ships under a versioned path and is not on PATH in the dev container. @@ -34,12 +35,49 @@ postgresql_proc = factories.postgresql_proc( postgresql = factories.postgresql("postgresql_proc") +def _create_pg_enum_types(engine: Engine) -> None: + """Emit CREATE TYPE for PostgreSQL enum types used by UploadedFile. + + SQLModel.metadata.create_all uses create_type=False for these enums + (they are normally created by Alembic migrations). Tests need them upfront. + A DO block swallows duplicate_object so the fixture is safe to call on a + pre-seeded database. + """ + from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum + + ft_values = ", ".join(f"'{e.value}'" for e in FileTypeEnum) + fs_values = ", ".join(f"'{e.value}'" for e in FileSourceEnum) + with engine.connect() as conn: + conn.execute( + text( + f""" + DO $$ BEGIN + CREATE TYPE file_type AS ENUM ({ft_values}); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + """ + ) + ) + conn.execute( + text( + f""" + DO $$ BEGIN + CREATE TYPE file_source AS ENUM ({fs_values}); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + """ + ) + ) + conn.commit() + + @pytest.fixture def db_engine(postgresql: Connection[Any]) -> Iterator[Engine]: """A SQLModel engine bound to a fresh, ephemeral PostgreSQL database.""" info = postgresql.info url = f"postgresql+psycopg://{info.user}:@{info.host}:{info.port}/{info.dbname}" engine = create_engine(url) + _create_pg_enum_types(engine) SQLModel.metadata.create_all(engine) try: yield engine diff --git a/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py b/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py index c688008e..9c65b6c9 100644 --- a/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py +++ b/tests/orchestration/magic_plan/test_magic_plan_orchestrator.py @@ -9,7 +9,7 @@ from domain.magicplan.api.response import MagicPlanPlan, PlanSummary from domain.magicplan.mapper import map_plan from domain.magicplan.models import Plan -from backend.app.db.models.uploaded_file import ( +from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, UploadedFile, From 53f0da8666d94860131d96882d0ece0824bae8b5 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:53:25 +0000 Subject: [PATCH 06/18] =?UTF-8?q?UploadedFilePostgresRepository=20returns?= =?UTF-8?q?=20latest=20uploaded=20file=20by=20deal=20ID=20and=20type=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- repositories/uploaded_file/__init__.py | 0 .../uploaded_file_postgres_repository.py | 20 +++++ tests/repositories/uploaded_file/__init__.py | 0 .../test_uploaded_file_postgres_repository.py | 80 +++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 repositories/uploaded_file/__init__.py create mode 100644 repositories/uploaded_file/uploaded_file_postgres_repository.py create mode 100644 tests/repositories/uploaded_file/__init__.py create mode 100644 tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py diff --git a/repositories/uploaded_file/__init__.py b/repositories/uploaded_file/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repositories/uploaded_file/uploaded_file_postgres_repository.py b/repositories/uploaded_file/uploaded_file_postgres_repository.py new file mode 100644 index 00000000..6936e14d --- /dev/null +++ b/repositories/uploaded_file/uploaded_file_postgres_repository.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Optional + +from sqlmodel import Session + +from infrastructure.postgres.uploaded_file_table import FileTypeEnum, UploadedFile + + +class UploadedFilePostgresRepository: + def __init__(self, session: Session) -> None: + self._session = session + + def get_latest_by_hubspot_deal_id( + self, hubspot_deal_id: str, file_type: FileTypeEnum + ) -> Optional[UploadedFile]: + raise NotImplementedError + + def insert(self, uploaded_file: UploadedFile) -> None: + raise NotImplementedError diff --git a/tests/repositories/uploaded_file/__init__.py b/tests/repositories/uploaded_file/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py b/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py new file mode 100644 index 00000000..5300c020 --- /dev/null +++ b/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import Engine +from sqlmodel import Session + +from infrastructure.postgres.uploaded_file_table import FileTypeEnum, UploadedFile +from repositories.uploaded_file.uploaded_file_postgres_repository import ( + UploadedFilePostgresRepository, +) + +_DEAL_ID = "deal-abc-123" +_BUCKET = "test-bucket" + + +def _make_uploaded_file( + hubspot_deal_id: str = _DEAL_ID, + file_type: FileTypeEnum = FileTypeEnum.MAGIC_PLAN_JSON, + offset_seconds: int = 0, +) -> UploadedFile: + return UploadedFile( + s3_file_bucket=_BUCKET, + s3_file_key=f"documents/{hubspot_deal_id}/plan.json", + s3_upload_timestamp=datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + + timedelta(seconds=offset_seconds), + hubspot_deal_id=hubspot_deal_id, + file_type=file_type.value, + ) + + +def test_returns_most_recent_row_by_timestamp(db_engine: Engine) -> None: + # Arrange — two rows for the same deal/type; older first, newer second + older = _make_uploaded_file(offset_seconds=0) + newer = _make_uploaded_file(offset_seconds=60) + with Session(db_engine) as session: + session.add(older) + session.add(newer) + session.commit() + newer_id = newer.id + + # Act + with Session(db_engine) as session: + result = UploadedFilePostgresRepository(session).get_latest_by_hubspot_deal_id( + _DEAL_ID, FileTypeEnum.MAGIC_PLAN_JSON + ) + + # Assert + assert result is not None + assert result.id == newer_id + + +def test_returns_none_when_no_matching_row(db_engine: Engine) -> None: + # Arrange — empty database + + # Act + with Session(db_engine) as session: + result = UploadedFilePostgresRepository(session).get_latest_by_hubspot_deal_id( + "nonexistent-deal", FileTypeEnum.MAGIC_PLAN_JSON + ) + + # Assert + assert result is None + + +def test_does_not_return_row_with_different_file_type(db_engine: Engine) -> None: + # Arrange — row exists but for a different file_type + row = _make_uploaded_file(file_type=FileTypeEnum.OTHER) + with Session(db_engine) as session: + session.add(row) + session.commit() + + # Act + with Session(db_engine) as session: + result = UploadedFilePostgresRepository(session).get_latest_by_hubspot_deal_id( + _DEAL_ID, FileTypeEnum.MAGIC_PLAN_JSON + ) + + # Assert + assert result is None From f08a75e1032abdddca8d88661ff0f5eb5b0740d8 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:54:14 +0000 Subject: [PATCH 07/18] =?UTF-8?q?UploadedFilePostgresRepository=20returns?= =?UTF-8?q?=20latest=20uploaded=20file=20by=20deal=20ID=20and=20type=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../uploaded_file_postgres_repository.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/repositories/uploaded_file/uploaded_file_postgres_repository.py b/repositories/uploaded_file/uploaded_file_postgres_repository.py index 6936e14d..9bf4a3d3 100644 --- a/repositories/uploaded_file/uploaded_file_postgres_repository.py +++ b/repositories/uploaded_file/uploaded_file_postgres_repository.py @@ -2,7 +2,8 @@ from __future__ import annotations from typing import Optional -from sqlmodel import Session +from sqlalchemy import select +from sqlmodel import Session, col from infrastructure.postgres.uploaded_file_table import FileTypeEnum, UploadedFile @@ -14,7 +15,14 @@ class UploadedFilePostgresRepository: def get_latest_by_hubspot_deal_id( self, hubspot_deal_id: str, file_type: FileTypeEnum ) -> Optional[UploadedFile]: - raise NotImplementedError + stmt = ( + select(UploadedFile) + .where(col(UploadedFile.hubspot_deal_id) == hubspot_deal_id) + .where(col(UploadedFile.file_type) == file_type.value) + .order_by(col(UploadedFile.s3_upload_timestamp).desc()) + .limit(1) + ) + return self._session.execute(stmt).scalars().one_or_none() # pyright: ignore[reportDeprecated] def insert(self, uploaded_file: UploadedFile) -> None: - raise NotImplementedError + self._session.add(uploaded_file) From a1d09aa88046fb238b7b32f5deb9920f947d7667 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 11:59:09 +0000 Subject: [PATCH 08/18] =?UTF-8?q?Audit=20generator=20populates=20XLSX,=20u?= =?UTF-8?q?ploads=20to=20S3,=20and=20records=20UploadedFile=20row=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- applications/audit-generator/handler.py | 1 + .../audit_generator_trigger_request.py | 1 + applications/audit_generator/__init__.py | 0 .../audit_generator_trigger_request.py | 9 + applications/audit_generator/handler.py | 40 ++++ orchestration/audit_generator_orchestrator.py | 24 ++ orchestration/audit_generator_unit_of_work.py | 39 ++++ .../applications/audit_generator/__init__.py | 0 .../test_audit_generator_handler.py | 94 ++++++++ .../orchestration/audit_generator/__init__.py | 0 .../test_audit_generator_orchestrator.py | 219 ++++++++++++++++++ 11 files changed, 427 insertions(+) create mode 100644 applications/audit_generator/__init__.py create mode 100644 applications/audit_generator/audit_generator_trigger_request.py create mode 100644 applications/audit_generator/handler.py create mode 100644 orchestration/audit_generator_orchestrator.py create mode 100644 orchestration/audit_generator_unit_of_work.py create mode 100644 tests/applications/audit_generator/__init__.py create mode 100644 tests/applications/audit_generator/test_audit_generator_handler.py create mode 100644 tests/orchestration/audit_generator/__init__.py create mode 100644 tests/orchestration/audit_generator/test_audit_generator_orchestrator.py diff --git a/applications/audit-generator/handler.py b/applications/audit-generator/handler.py index e69de29b..6ad89ccb 100644 --- a/applications/audit-generator/handler.py +++ b/applications/audit-generator/handler.py @@ -0,0 +1 @@ +from applications.audit_generator.handler import handler # noqa: F401 diff --git a/applications/audit-generator/handler/audit_generator_trigger_request.py b/applications/audit-generator/handler/audit_generator_trigger_request.py index e69de29b..165a8e6c 100644 --- a/applications/audit-generator/handler/audit_generator_trigger_request.py +++ b/applications/audit-generator/handler/audit_generator_trigger_request.py @@ -0,0 +1 @@ +from applications.audit_generator.audit_generator_trigger_request import AuditGeneratorTriggerRequest # noqa: F401 diff --git a/applications/audit_generator/__init__.py b/applications/audit_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/applications/audit_generator/audit_generator_trigger_request.py b/applications/audit_generator/audit_generator_trigger_request.py new file mode 100644 index 00000000..042db001 --- /dev/null +++ b/applications/audit_generator/audit_generator_trigger_request.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel, ConfigDict + + +class AuditGeneratorTriggerRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + task_id: str + sub_task_id: str + hubspot_deal_id: str diff --git a/applications/audit_generator/handler.py b/applications/audit_generator/handler.py new file mode 100644 index 00000000..b560f808 --- /dev/null +++ b/applications/audit_generator/handler.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import os +from typing import Any + +import boto3 + +from applications.audit_generator.audit_generator_trigger_request import ( + AuditGeneratorTriggerRequest, +) +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine, make_session +from infrastructure.s3.s3_client import S3Client +from orchestration.audit_generator_orchestrator import AuditGeneratorOrchestrator +from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork +from utilities.aws_lambda.subtask_handler import subtask_handler + +_engine = make_engine(PostgresConfig.from_env(os.environ)) + + +@subtask_handler() +def handler(body: dict[str, Any], context: Any) -> None: + trigger = AuditGeneratorTriggerRequest.model_validate(body) + + boto3_client: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + boto_s3: Any = boto3_client("s3") + bucket = os.environ["S3_BUCKET_NAME"] + s3_client = S3Client(boto_s3_client=boto_s3, bucket=bucket) + + def session_factory() -> Any: + return make_session(_engine) + + def uow_factory() -> AuditGeneratorUnitOfWork: + return AuditGeneratorUnitOfWork(session_factory) + + AuditGeneratorOrchestrator( + hubspot_deal_id=trigger.hubspot_deal_id, + s3_client=s3_client, + uow_factory=uow_factory, + ).run() diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py new file mode 100644 index 00000000..887f0092 --- /dev/null +++ b/orchestration/audit_generator_orchestrator.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from infrastructure.s3.s3_client import S3Client + +if TYPE_CHECKING: + from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork + + +class AuditGeneratorOrchestrator: + def __init__( + self, + hubspot_deal_id: str, + s3_client: S3Client, + uow_factory: Callable[[], "AuditGeneratorUnitOfWork"], + ) -> None: + self._hubspot_deal_id = hubspot_deal_id + self._s3_client = s3_client + self._uow_factory = uow_factory + + def run(self) -> None: + raise NotImplementedError diff --git a/orchestration/audit_generator_unit_of_work.py b/orchestration/audit_generator_unit_of_work.py new file mode 100644 index 00000000..694d6039 --- /dev/null +++ b/orchestration/audit_generator_unit_of_work.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from collections.abc import Callable +from types import TracebackType +from typing import Optional + +from sqlmodel import Session + +from repositories.magic_plan.magic_plan_postgres_repository import ( + MagicPlanPostgresRepository, +) +from repositories.uploaded_file.uploaded_file_postgres_repository import ( + UploadedFilePostgresRepository, +) + + +class AuditGeneratorUnitOfWork: + def __init__(self, session_factory: Callable[[], Session]) -> None: + self._session_factory = session_factory + + def __enter__(self) -> "AuditGeneratorUnitOfWork": + self._session = self._session_factory() + self.uploaded_file = UploadedFilePostgresRepository(self._session) + self.magic_plan = MagicPlanPostgresRepository(self._session) + return self + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + try: + self._session.rollback() + finally: + self._session.close() + + def commit(self) -> None: + self._session.commit() diff --git a/tests/applications/audit_generator/__init__.py b/tests/applications/audit_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/applications/audit_generator/test_audit_generator_handler.py b/tests/applications/audit_generator/test_audit_generator_handler.py new file mode 100644 index 00000000..f9bc74b6 --- /dev/null +++ b/tests/applications/audit_generator/test_audit_generator_handler.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from applications.audit_generator.handler import handler + +_ENV = { + "DATABASE_URL": "postgresql+psycopg://user:pass@localhost/db", + "S3_BUCKET_NAME": "test-bucket", + # PostgresConfig.from_env also reads these individual vars; DATABASE_URL is + # used directly when constructing the engine in the handler module scope, so + # we patch make_engine instead of the env. +} + +_VALID_BODY: dict[str, Any] = { + "task_id": "task-1", + "sub_task_id": "subtask-1", + "hubspot_deal_id": "deal-xyz", +} + + +def _call(body: dict[str, Any]) -> Any: + return handler.__wrapped__(body, None) # type: ignore[attr-defined] + + +# --- request validation --- + + +def test_invalid_body_raises_validation_error() -> None: + # Arrange — body missing all required fields + body: dict[str, Any] = {} + + # Act / Assert + with patch("applications.audit_generator.handler.AuditGeneratorOrchestrator"): + with pytest.raises(ValidationError): + _call(body) + + +# --- orchestrator construction --- + + +def test_handler_passes_hubspot_deal_id_from_body_to_orchestrator() -> None: + # Arrange + mock_orch = MagicMock() + mock_orch.run.return_value = None + + # Act + with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.S3Client") as MockS3, \ + patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch) as MockOrch: + MockS3.return_value = MagicMock() + _call(_VALID_BODY) + + # Assert — deal id flows from body into the orchestrator constructor + MockOrch.assert_called_once() + assert MockOrch.call_args.kwargs["hubspot_deal_id"] == "deal-xyz" + + +def test_handler_passes_bucket_from_env_to_s3_client() -> None: + # Arrange + mock_orch = MagicMock() + mock_orch.run.return_value = None + + # Act + with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.S3Client") as MockS3, \ + patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch): + _call(_VALID_BODY) + + # Assert — bucket name from env reaches S3Client constructor + MockS3.assert_called_once() + assert MockS3.call_args.kwargs["bucket"] == "test-bucket" + + +# --- return value --- + + +def test_handler_returns_none_on_success() -> None: + # Arrange + mock_orch = MagicMock() + mock_orch.run.return_value = None + + # Act + with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.S3Client"), \ + patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch): + result = _call(_VALID_BODY) + + # Assert + assert result is None diff --git a/tests/orchestration/audit_generator/__init__.py b/tests/orchestration/audit_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py b/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py new file mode 100644 index 00000000..18445fe5 --- /dev/null +++ b/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from io import BytesIO +from typing import Any +from unittest.mock import MagicMock, call, patch + +import pytest + +from domain.magicplan.models import ( + Door, + DoorVentilation, + Floor, + Plan, + Room, + Window, + WindowVentilation, +) +from infrastructure.postgres.uploaded_file_table import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) +from infrastructure.s3.s3_client import S3Client +from orchestration.audit_generator_orchestrator import AuditGeneratorOrchestrator + +_DEAL_ID = "deal-abc" +_BUCKET = "test-bucket" +_EXPECTED_S3_KEY = f"documents/hubspot_deal_id/{_DEAL_ID}/ventilation_audit.xlsx" + + +def _make_window(with_ventilation: bool = True) -> Window: + vent = ( + WindowVentilation( + opening_type="Hinged", + num_openings=1, + pct_openable=50, + trickle_vent_area_mm2=1000, + num_trickle_vents=2, + ) + if with_ventilation + else None + ) + return Window(width_m=1.0, height_m=1.2, area_m2=1.2, ventilation=vent) + + +def _make_door(with_ventilation: bool = True) -> Door: + vent = DoorVentilation(undercut_mm=10.0) if with_ventilation else None + return Door(width_mm=800.0, height_mm=2000.0, ventilation=vent) + + +def _make_plan( + num_rooms: int = 1, + num_windows_per_room: int = 1, + num_doors_per_room: int = 1, +) -> Plan: + rooms = [ + Room( + name=f"Room {i}", + width_m=3.0, + length_m=4.0, + area_m2=12.0, + windows=[_make_window() for _ in range(num_windows_per_room)], + doors=[_make_door() for _ in range(num_doors_per_room)], + ) + for i in range(num_rooms) + ] + return Plan( + uid="test-uid", + name="Test Plan", + address="1 Test St", + postcode="TE1 1ST", + floors=[Floor(level=0, name="Ground", rooms=rooms)], + ) + + +def _make_uploaded_file_row(id: int = 1) -> UploadedFile: + return UploadedFile( + id=id, + s3_file_bucket=_BUCKET, + s3_file_key="documents/deal/plan.json", + s3_upload_timestamp=None, # type: ignore[arg-type] + hubspot_deal_id=_DEAL_ID, + file_type=FileTypeEnum.MAGIC_PLAN_JSON.value, + ) + + +def _make_mock_uow( + uploaded_file_row: Any = None, + plan: Any = None, +) -> tuple[MagicMock, MagicMock]: + """Return (mock_uow, mock_uow_factory).""" + mock_uow = MagicMock() + mock_uow.__enter__ = MagicMock(return_value=mock_uow) + mock_uow.__exit__ = MagicMock(return_value=False) + mock_uow.uploaded_file.get_latest_by_hubspot_deal_id.return_value = uploaded_file_row + mock_uow.magic_plan.get_plan_by_uploaded_file_id.return_value = plan + mock_uow_factory = MagicMock(return_value=mock_uow) + return mock_uow, mock_uow_factory + + +def _make_s3() -> MagicMock: + s3 = MagicMock(spec=S3Client) + s3.bucket = _BUCKET + return s3 + + +def _make_orchestrator( + s3: Any = None, + uow_factory: Any = None, + deal_id: str = _DEAL_ID, +) -> AuditGeneratorOrchestrator: + return AuditGeneratorOrchestrator( + hubspot_deal_id=deal_id, + s3_client=s3 or _make_s3(), + uow_factory=uow_factory or MagicMock(), + ) + + +# --- error: no uploaded file --- + + +def test_raises_when_no_magic_plan_json_uploaded() -> None: + # Arrange + mock_uow, mock_uow_factory = _make_mock_uow(uploaded_file_row=None) + orch = _make_orchestrator(uow_factory=mock_uow_factory) + + # Act / Assert + with pytest.raises(ValueError, match="No MagicPlan"): + orch.run() + + +# --- error: plan not yet parsed --- + + +def test_raises_when_plan_not_yet_parsed() -> None: + # Arrange + mock_uow, mock_uow_factory = _make_mock_uow( + uploaded_file_row=_make_uploaded_file_row(), plan=None + ) + orch = _make_orchestrator(uow_factory=mock_uow_factory) + + # Act / Assert + with pytest.raises(ValueError, match="not yet parsed"): + orch.run() + + +# --- happy path --- + + +def test_uploads_to_correct_s3_key() -> None: + # Arrange + s3 = _make_s3() + plan = _make_plan() + mock_uow, mock_uow_factory = _make_mock_uow( + uploaded_file_row=_make_uploaded_file_row(), plan=plan + ) + orch = _make_orchestrator(s3=s3, uow_factory=mock_uow_factory) + + # Act + orch.run() + + # Assert + s3.put_object.assert_called_once() + assert s3.put_object.call_args.args[0] == _EXPECTED_S3_KEY + + +def test_inserts_uploaded_file_with_correct_enums() -> None: + # Arrange + plan = _make_plan() + mock_uow, mock_uow_factory = _make_mock_uow( + uploaded_file_row=_make_uploaded_file_row(), plan=plan + ) + orch = _make_orchestrator(uow_factory=mock_uow_factory) + + # Act + orch.run() + + # Assert — the UploadedFile inserted has the correct type/source + mock_uow.uploaded_file.insert.assert_called_once() + inserted: UploadedFile = mock_uow.uploaded_file.insert.call_args.args[0] + assert inserted.file_type == FileTypeEnum.VENTILATION_AUDIT.value + assert inserted.file_source == FileSourceEnum.AUDIT_GENERATOR.value + assert inserted.hubspot_deal_id == _DEAL_ID + assert inserted.s3_file_key == _EXPECTED_S3_KEY + + +def test_commits_after_s3_upload() -> None: + # Arrange + s3 = _make_s3() + plan = _make_plan() + mock_uow, mock_uow_factory = _make_mock_uow( + uploaded_file_row=_make_uploaded_file_row(), plan=plan + ) + call_order: list[str] = [] + s3.put_object.side_effect = lambda *a, **kw: call_order.append("s3_upload") + mock_uow.commit.side_effect = lambda: call_order.append("commit") + orch = _make_orchestrator(s3=s3, uow_factory=mock_uow_factory) + + # Act + orch.run() + + # Assert — S3 upload happens before DB commit + assert call_order == ["s3_upload", "commit"] + + +# --- 50-row limit --- + + +def test_raises_when_series_exceeds_50_rows() -> None: + # Arrange — 51 windows, which exceeds the template limit + plan = _make_plan(num_rooms=1, num_windows_per_room=51) + mock_uow, mock_uow_factory = _make_mock_uow( + uploaded_file_row=_make_uploaded_file_row(), plan=plan + ) + orch = _make_orchestrator(uow_factory=mock_uow_factory) + + # Act / Assert + with pytest.raises(ValueError, match="50"): + orch.run() From 612d522b353008b14619451bf0ed326fdf5f0e48 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 12:01:08 +0000 Subject: [PATCH 09/18] =?UTF-8?q?Audit=20generator=20populates=20XLSX,=20u?= =?UTF-8?q?ploads=20to=20S3,=20and=20records=20UploadedFile=20row=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- orchestration/audit_generator_orchestrator.py | 112 +++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py index 887f0092..818784c1 100644 --- a/orchestration/audit_generator_orchestrator.py +++ b/orchestration/audit_generator_orchestrator.py @@ -1,13 +1,86 @@ from __future__ import annotations from collections.abc import Callable -from typing import TYPE_CHECKING +from datetime import datetime, timezone +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional +import openpyxl + +from domain.magicplan.models import Door, Plan, Room, Window +from infrastructure.postgres.uploaded_file_table import ( + FileSourceEnum, + FileTypeEnum, + UploadedFile, +) from infrastructure.s3.s3_client import S3Client if TYPE_CHECKING: from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork +_TEMPLATE_PATH = Path(__file__).parent.parent / "applications" / "audit-generator" / "d1_ventilation_template.xlsx" +_SHEET_NAME = "D1 Ventilation" +_DATA_START_ROW = 6 +_MAX_ROWS = 50 + + +def _write_cell(sheet: Any, row: int, col: str, value: Any) -> None: + sheet[f"{col}{row}"] = value + + +def _populate_sheet(sheet: Any, plan: Plan) -> None: + rooms: list[Room] = [room for floor in plan.floors for room in floor.rooms] + windows: list[tuple[str, Window]] = [ + (room.name, w) for room in rooms for w in room.windows + ] + doors: list[tuple[str, Door]] = [ + (room.name, d) for room in rooms for d in room.doors + ] + + if len(rooms) > _MAX_ROWS: + raise ValueError(f"Room series exceeds {_MAX_ROWS} rows ({len(rooms)} rooms)") + if len(windows) > _MAX_ROWS: + raise ValueError(f"Window series exceeds {_MAX_ROWS} rows ({len(windows)} windows)") + if len(doors) > _MAX_ROWS: + raise ValueError(f"Door series exceeds {_MAX_ROWS} rows ({len(doors)} doors)") + + for i, room in enumerate(rooms): + row = _DATA_START_ROW + i + _write_cell(sheet, row, "B", room.name) + _write_cell(sheet, row, "D", room.area_m2) + + for i, (room_name, window) in enumerate(windows): + row = _DATA_START_ROW + i + vent = window.ventilation + _write_cell(sheet, row, "G", room_name) + _write_cell(sheet, row, "H", window.width_m) + _write_cell(sheet, row, "I", window.height_m) + # J = formula =H*I — do not write + _write_cell(sheet, row, "K", vent.opening_type if vent else 0) + _write_cell(sheet, row, "L", vent.num_openings if vent else 0) + pct = vent.pct_openable if vent else None + _write_cell(sheet, row, "M", (pct / 100) if pct is not None else 0) + # N = formula =J*M — do not write + # O, P = blank (visual check by auditor) + _write_cell(sheet, row, "Q", vent.trickle_vent_area_mm2 if vent else 0) + _write_cell(sheet, row, "R", vent.num_trickle_vents if vent else 0) + # S = formula =Q*R — do not write + + for i, (room_name, door) in enumerate(doors): + row = _DATA_START_ROW + i + vent = door.ventilation + _write_cell(sheet, row, "V", room_name) + _write_cell(sheet, row, "W", door.width_mm) + _write_cell(sheet, row, "X", vent.undercut_mm if vent else 0) + # Y = formula =W*X — do not write + + +def _serialise_workbook(wb: Any) -> bytes: + buf = BytesIO() + wb.save(buf) + return buf.getvalue() + class AuditGeneratorOrchestrator: def __init__( @@ -21,4 +94,39 @@ class AuditGeneratorOrchestrator: self._uow_factory = uow_factory def run(self) -> None: - raise NotImplementedError + with self._uow_factory() as uow: + uploaded_file = uow.uploaded_file.get_latest_by_hubspot_deal_id( + self._hubspot_deal_id, FileTypeEnum.MAGIC_PLAN_JSON + ) + if uploaded_file is None: + raise ValueError( + f"No MagicPlan JSON has been uploaded for deal {self._hubspot_deal_id!r}" + ) + + plan = uow.magic_plan.get_plan_by_uploaded_file_id(uploaded_file.id) + if plan is None: + raise ValueError( + f"MagicPlan JSON exists for deal {self._hubspot_deal_id!r} " + "but the plan is not yet parsed into the database" + ) + + wb = openpyxl.load_workbook(_TEMPLATE_PATH) + sheet = wb[_SHEET_NAME] + _populate_sheet(sheet, plan) + xlsx_bytes = _serialise_workbook(wb) + + s3_key = ( + f"documents/hubspot_deal_id/{self._hubspot_deal_id}/ventilation_audit.xlsx" + ) + self._s3_client.put_object(s3_key, xlsx_bytes) + + new_row = UploadedFile( + s3_file_bucket=self._s3_client.bucket, + s3_file_key=s3_key, + s3_upload_timestamp=datetime.now(timezone.utc), + hubspot_deal_id=self._hubspot_deal_id, + file_type=FileTypeEnum.VENTILATION_AUDIT.value, + file_source=FileSourceEnum.AUDIT_GENERATOR.value, + ) + uow.uploaded_file.insert(new_row) + uow.commit() From 82156fac8f8e9768f75647cabb141333568a4cc1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 12:05:55 +0000 Subject: [PATCH 10/18] =?UTF-8?q?Audit=20generator=20populates=20XLSX,=20u?= =?UTF-8?q?ploads=20to=20S3,=20and=20records=20UploadedFile=20row=20?= =?UTF-8?q?=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- applications/audit-generator/handler/requirements.txt | 1 + orchestration/audit_generator_orchestrator.py | 4 ++-- tests/conftest.py | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/applications/audit-generator/handler/requirements.txt b/applications/audit-generator/handler/requirements.txt index e69de29b..794cc3d3 100644 --- a/applications/audit-generator/handler/requirements.txt +++ b/applications/audit-generator/handler/requirements.txt @@ -0,0 +1 @@ +openpyxl diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py index 818784c1..6a1d529f 100644 --- a/orchestration/audit_generator_orchestrator.py +++ b/orchestration/audit_generator_orchestrator.py @@ -4,7 +4,7 @@ from collections.abc import Callable from datetime import datetime, timezone from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, cast import openpyxl @@ -103,7 +103,7 @@ class AuditGeneratorOrchestrator: f"No MagicPlan JSON has been uploaded for deal {self._hubspot_deal_id!r}" ) - plan = uow.magic_plan.get_plan_by_uploaded_file_id(uploaded_file.id) + plan = uow.magic_plan.get_plan_by_uploaded_file_id(cast(int, uploaded_file.id)) if plan is None: raise ValueError( f"MagicPlan JSON exists for deal {self._hubspot_deal_id!r} " diff --git a/tests/conftest.py b/tests/conftest.py index 17c2eefd..ee6d54f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,9 +29,7 @@ import infrastructure.postgres.uploaded_file_table as _uf_table # pyright: igno # pg_ctl ships under a versioned path and is not on PATH in the dev container. _PG_CTL = next(iter(sorted(glob.glob("/usr/lib/postgresql/*/bin/pg_ctl"))), "pg_ctl") -postgresql_proc = factories.postgresql_proc( - executable=_PG_CTL -) # pyright: ignore[reportUnknownMemberType] +postgresql_proc = factories.postgresql_proc(executable=_PG_CTL) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] postgresql = factories.postgresql("postgresql_proc") From e010fa1b40431f945e8bf1cf56edd2388d6e3ac7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 12:52:04 +0000 Subject: [PATCH 11/18] remove redundant files --- applications/audit-generator/handler.py | 1 - applications/audit-generator/handler/Dockerfile | 0 .../handler/audit_generator_trigger_request.py | 1 - .../audit-generator/handler/requirements.txt | 1 - .../d1_ventilation_template.xlsx | Bin 5 files changed, 3 deletions(-) delete mode 100644 applications/audit-generator/handler.py delete mode 100644 applications/audit-generator/handler/Dockerfile delete mode 100644 applications/audit-generator/handler/audit_generator_trigger_request.py delete mode 100644 applications/audit-generator/handler/requirements.txt rename applications/{audit-generator => audit_generator}/d1_ventilation_template.xlsx (100%) diff --git a/applications/audit-generator/handler.py b/applications/audit-generator/handler.py deleted file mode 100644 index 6ad89ccb..00000000 --- a/applications/audit-generator/handler.py +++ /dev/null @@ -1 +0,0 @@ -from applications.audit_generator.handler import handler # noqa: F401 diff --git a/applications/audit-generator/handler/Dockerfile b/applications/audit-generator/handler/Dockerfile deleted file mode 100644 index e69de29b..00000000 diff --git a/applications/audit-generator/handler/audit_generator_trigger_request.py b/applications/audit-generator/handler/audit_generator_trigger_request.py deleted file mode 100644 index 165a8e6c..00000000 --- a/applications/audit-generator/handler/audit_generator_trigger_request.py +++ /dev/null @@ -1 +0,0 @@ -from applications.audit_generator.audit_generator_trigger_request import AuditGeneratorTriggerRequest # noqa: F401 diff --git a/applications/audit-generator/handler/requirements.txt b/applications/audit-generator/handler/requirements.txt deleted file mode 100644 index 794cc3d3..00000000 --- a/applications/audit-generator/handler/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -openpyxl diff --git a/applications/audit-generator/d1_ventilation_template.xlsx b/applications/audit_generator/d1_ventilation_template.xlsx similarity index 100% rename from applications/audit-generator/d1_ventilation_template.xlsx rename to applications/audit_generator/d1_ventilation_template.xlsx From f8c955b2d3cd6eba579fac31a4ebda7bcf0cd22f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 13:03:05 +0000 Subject: [PATCH 12/18] local runner and correct template path --- orchestration/audit_generator_orchestrator.py | 2 +- scripts/run_audit_generator_local.py | 69 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 scripts/run_audit_generator_local.py diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py index 6a1d529f..1cea614f 100644 --- a/orchestration/audit_generator_orchestrator.py +++ b/orchestration/audit_generator_orchestrator.py @@ -19,7 +19,7 @@ from infrastructure.s3.s3_client import S3Client if TYPE_CHECKING: from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork -_TEMPLATE_PATH = Path(__file__).parent.parent / "applications" / "audit-generator" / "d1_ventilation_template.xlsx" +_TEMPLATE_PATH = Path(__file__).parent.parent / "applications" / "audit_generator" / "d1_ventilation_template.xlsx" _SHEET_NAME = "D1 Ventilation" _DATA_START_ROW = 6 _MAX_ROWS = 50 diff --git a/scripts/run_audit_generator_local.py b/scripts/run_audit_generator_local.py new file mode 100644 index 00000000..748e3f89 --- /dev/null +++ b/scripts/run_audit_generator_local.py @@ -0,0 +1,69 @@ +""" +Run audit_generator locally. Writes XLSX to ./local_output/ instead of S3. + +Usage: + cd /workspaces/model + python scripts/run_audit_generator_local.py +""" + +from __future__ import annotations + +import os +import sys +from io import BytesIO +from pathlib import Path +from typing import Any + +# Load .env before importing infra modules +from dotenv import load_dotenv + +load_dotenv(Path(__file__).parent.parent / "backend" / ".env") + +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine, make_session +from orchestration.audit_generator_orchestrator import AuditGeneratorOrchestrator +from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork + + +class _LocalS3Client: + """Writes to local filesystem instead of S3.""" + + def __init__(self, output_dir: Path) -> None: + self._output_dir = output_dir + self._output_dir.mkdir(parents=True, exist_ok=True) + + @property + def bucket(self) -> str: + return "local" + + def get_object(self, key: str) -> bytes: + raise NotImplementedError + + def put_object(self, key: str, body: bytes) -> str: + dest = self._output_dir / Path(key).name + dest.write_bytes(body) + print(f"Saved: {dest}") + return str(dest) + + +def main() -> None: + deal_id = sys.argv[1] if len(sys.argv) > 1 else input("hubspot_deal_id: ").strip() + output_dir = Path(__file__).parent.parent / "local_output" + + engine = make_engine(PostgresConfig.from_env(os.environ)) + + def session_factory() -> Any: + return make_session(engine) + + def uow_factory() -> AuditGeneratorUnitOfWork: + return AuditGeneratorUnitOfWork(session_factory) + + AuditGeneratorOrchestrator( + hubspot_deal_id=deal_id, + s3_client=_LocalS3Client(output_dir), # type: ignore[arg-type] + uow_factory=uow_factory, + ).run() + + +if __name__ == "__main__": + main() From 608006d9774c32cfd2faf229989c21f1f67774dd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 13:29:11 +0000 Subject: [PATCH 13/18] reapply conditional formatting to populated file --- .../d1_ventilation_template.xlsx | Bin 15658 -> 16133 bytes orchestration/audit_generator_orchestrator.py | 26 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/applications/audit_generator/d1_ventilation_template.xlsx b/applications/audit_generator/d1_ventilation_template.xlsx index 057da026fc51d3b5e3abe2ca43732e8d8252c1d3..f42272e0efd71fa3f688a21288d2489580d83029 100644 GIT binary patch delta 7483 zcmcgx2{e>%+b1F=TO_hKjOedk-wH|Ka}p69yn-+f)b>$-lo^W1sYBf^-C_37w2X(*o! z-x6jmhCLg;WhM*<_E5g<_C+!P8@{bJhPsye8hdDH==agkPzDEj%KEuEpk16z{n75; zm;7V`y*vw$?cRfO%wb%bjo9hpKHU=UzZ|(yy%!@h4`R1ckttCTeVNc!_tNe{)mff^ zVZ~{aHAvR?0Ld;X)6PA&X-z~l!p5Nkb#HB z_#VCDrI!()udw*q&u0ac6vPA&{PmRO#vv9?sR}%W;fq*5Z!-lVyNrpdMof%0gN#2` zELY9;o77{|pB9sa8ljp)&kd8S<#;vnKFEu9`I1$8WWV6`Y~TI}%h7n8?Ao!uVp}pA zg31)E$*Vj^cjD&lar5@;sG5qj<;ya#BrUp|sZMk&MRsol(R!{xVqtQ#gfBkG*`77< zMAF5vk9@sMeom(ip~bH_t-}DgAcsxk54%&=xo=Gwm$@D9M~+^3S2uN|*Z!i6-y_uN zdPzl*?^cb){?4P?yjseO=88$hLy~b~rbjVi9TWO*U#2LO!x9aCFp)y=D8tIvCpb2t zPdI1CI~eCtJajLIfCE}ISMmza-FfqfCgfUlqu}d$- z>+7|0AGZ?;(u7Fl{uTZhfNr-I_kp$8YV&%^_vkoH86!IfEu|Qba_pnqM|s{8H7bXb z8HKeW4HEnzT`sm?p^D9))>rr>V5e|gZ#&57=7H36Q7O0SOr!T&`ya~ zW-PnuR2}c^3D%Mi2swbQdJ6RyG%rBDTh_65Y6MnT^J$^s&XzYX-3q$R=-?)HOy&6A z^tg#$rK~eP%m>9gzl@&3&P6O<#MhwY#1s*_fSk{v(~*Z>T|@7wDZE?luf4wIw6Qq9 zp&revciIz6C*fCr{Ke~#%S1-v3yxdL6G6xFC7R1T`?4a^_@3Km!yXAPbj#@+(e6}+ z$4IAZ;@~g!gXNG?YItZ8k!^N;2+rdxRE_a zfTlESzk!y4i0%DNE_t5Df~SyUwz_<(=8(aMj{InyVm?#aHw>`IRgdlEqP;^xYU$E9D?S6Hd zmy+#iL+(k5pzckaXU%qX;HKozMxQ!V7wBPMuozSxB1>5tU4Qj`8iq1_L7r4WMYWk$ zqYM@2rY3@~)O>+-HxE4_W$LVL!mq&6imKOHa0WPm$?oouc$@691eembB?F~a$7H7E zLGtSL{OZjx&yTadikqV5c^jNkb~BQZVTUnc#8pq6cX!ToNW2W;qz$Q3Ssg_N&^>SZ z(ABGRrRs<7_d5CxPuvY!x#QyP#b%~)1b<)})*9LoD1-?2OT2~dAR_lCsRibLxKYMC%EGB&u1rg~O zQ|POeYC^3n$e^Coy}|pky*KIdwBj3!56mE0c~X#8-EFrzV%*7?nm|{jwyg zKif1r8{R5dLz|WpwB(XQYG!W?W>l^bycINm*?Of4N4N|PYWo7c+mte6{b3~igzZ|H zdu_&JpJe-c!(Bgh)K@0tuOwSrilz;3S<#aXET$)QW0o_U{o~sKvE}ZKOyF?SP+ma7 zb+PaqgEgblwpU)bvkF96S(wA{NNm7?y4jfi{;gn~8mXdHR8>Y&8M%QfnD$5rD1i@W z4~~ac?<2?bFQgP!4ylP5btjyiAk?Xgp^N%qqd~-^zOO@ds$~gY@iHVxNh&F%V%u*m zc$qAVf?Wz{)#-7#zNjq%Kw}%V6;uR$I~cMahl$}8Cw(VYJHN$D+E2*CVo1STPz~JP zZp=C9O3!+0f$Ku#gDeZYg;&L)EHR9nP}euo>MO*p=<#>HgD0&*uj;K&9ChJG`=;t0 z6>CI%D30zgc5AGX;5+Q)g(4l>h8ysoOvyt#@M9bvx?bu-^cP11qs7l$BaxHEx09Q~ z^q=>C3IW!id1qzDZf79QB@^E9=2sQ?oipp07qqf9(2lsv*Lc!fX~CygxQU=tyx;kHVMNy@2gPpqVDYMu$DpDWYd(vT7Frp^_|*P*3|=jN_C%>p8W8Z z?Zo`3m9m-H8^ER>2k6+~Bc@7!lt_J!+glS7XGCH*MEMXKzT1s3juxtog~>9rG?3@_ z6-lkQjOwET#Sg$y`-qxmhsR2PT$N?MmV-=fYRu*IZMVm_o&u2ZaM(*TK|{s?6;)8pG^<+AoJqJTxJPYBYQkGQ^W0>D>5kaopr+ ztgq8^p%Knne}MYxXYkey<`i7dD{u&cH*t{0bXVB43O6~k=m+()w%YTtXr0T^Qa5Mj zNoT3bJ>rNj1Zb_JOTKM}#cE-5nk8(xqWQG$I_YuWw06mf`(h2d&D9)tD-hcJV=NJ2 zH$zH<`(ALjO(!Kf_+FUQ4n#WXU1~^h!sdLINW(J1&iQp%JDR8QTYtQVIJWyur4oYFByDO!2XUb0YC(z@NILqUd6Y~dC#%O9X0mM z<+EQ@hhJiJt;+bUM+yubhNlN7F7j})atiXPpC?s1-86gKoFX93EPsg_J@`_vWh*gbnqhA#009d*bUuL{mb-hiqXX`T3n&pNBNU45W;xII{pjZ zr1~b70gQ!rd;P3)%9uXu@w#gdof2K&ksA}WJ_2Xv2n~|b>zF86>ANRG1zyg3NNKzd z+gofe3NWu}`t_NS zu|<&cQ(I||e~={Y$A*&K*4pwH7H_O`j<9RRiNraI!sC{&utyYGu5$F8?Y1yUD=I3r zK)x+t;CUhR^|0&bhI`hDz6TV&-4zS-2lA|foLoLBoBNY8BrSALqMn~nwy+EJL;7Kp zD&)#8Hfgn5AV~ZH#-Wi-R~h>NnD%kaUh z!ag>*1eE7VA{rk>z}F8Lk)?ZtFh53Lf_}8fVVD)c%Y}dP!}S6DHjnea^TVco4D`c>;>UO*JeqPT z{3kyE=N04?DC{fb^CwB< ziecz5%q(&kx_<`}=6I4u{4g8m0d{NzbCc#BN8z(2S?qT@Lta<{mUxKCNZRjc1m3%%z+q4>xcf zn8XsABnR@mCTaA+$;BCxyDC}3lR&9CSR!TbHgSnSJqa8SWe6MX)eEcg*l@&w+{E6)0G466O z-PS65OcUVRa(&dgdG3ECTLx3Xcl*SM9Q~<#W(>`XHXP$}^y(#bVn0wDI-Vzh-Xifcpw$2;U`8Q^rxc0 z+}}wJ7lI16=-!FPx#cU#k3jxvgImaaKyx_0*4>BZxRpv@5GcO-<_2!0nr@melk+j> zAv!}%4;Yg>Xji{bVsX3alIRm>E(LK>L`y~-Hb(t?6zGIS1(Iv)fSN5sJwWyAV4k9Q zKAz_(F(Xw`!r55wt0qG|n1Z5?eH#hM)yxgM;Q3#zy%a2oy63?~Zs5+7AQ(=?^YWBS z1`2fc@4xxuJfpK|# z5@SfEw}t)HCTeA<$$V~lpO=OTA3ViG`)j&~-JH5OVJy^` zs9TOq0mt5=Qukkh0RM;rIsy7qo~mtK5k(7yQyM-h9>)fTOZCwWR1YZVUoZboRVIS# zAa?24iD4%kX((7npp%^xdNWB7Fhe_LE3n-`JAf0;+HRpo3M7b2skK$G@V^g(t zKk`)z@~8@Cc;A=g^w*jK;_I> z3gV#DS;!-Ls*+~ZFF-sDzbdkbyh{khp4eCte>+ z125<@#S91QUL6d#a#z>E2s^=QJnF--e|LI_0st6*OZ=}u(3JH&U{TLKK;CXqBpCtn z4~9&oouZ^hpnHX?uRDbG*!AQ=m&)UDouGXWB`p9L!%m8<3^NM`KqoUOTa+15_bDyK z4w@-kP@s4F_W}r4K!TKmc4`WDrvKXv|K|w?e$lnF)PI+V29Rwq5@9g0!Wd@tq&n9Y zFtcfJFL1wB;?dsOuhZa^>OGnB01lS*9m;+nTG1->ix5DsBzD000a%YcH*j(z5M~&Y zmSMvK9QcCl0<{Bqd^o5b=ni92{AH$gpz9wUm_!S808cRZm=5r|KfqNI47+^(6@d6)i_A`s4eyWj|j z&?wt<=5LVy+zEi{#s8@jL|seJNs!I)zuBbfAs8c753@;-ce`eQ(zIZFzXcJbEBya4 z8vuo4xv;y z`OgXm;q10wkwJ*&+0(m0{12Sy554^2Uh*5^XAe#u2l$u2pVmtd4?Q!vpKv)&J<}f( zd%p`B#}{W4=WSB0H7P)NFCHYmDjo`-q$SW0DF2HdAr$H5o!V}kaWExCwlrPIA(IXS zR@RBVFW8#Y=y4cvUpHCy*&^k@W_~4a~buU5zHzqLpbC^vEr-dW0YeP-&0^PbcK?199NL=6x&7^VVybYGC z%|e8Gd=xkwqC05TYvi2uGO~JdAwW*RRsJe@|`8?0z*!3KC0|)U#>a=RGxqI zMzl*q@Sf2&yK+PO)@gGWY5m?H7jZk;Izm#QPIefPHPv%|_`G|3WtJMj-P|Isb^{WO zy4KTuwRDcW4XKd@hC8zFLbg4;OX9(ogrWy2ujFQCO4b)wyo+%!DhH?e)UxZ$?H}Pj z4ZU~n{{AE-sI)D6b>`({VYdPBJgKA1o9vSEnYzY6fiw{f^us_y&Y{_l_!gKBn+0C{lH zaSaMA%t_tb!0m_LC0^Z97q`dBCAMEW>k?l`ZZ!|Pv>v8k6c7yUSe*olT4&$&yVhu^ z${v;9-oBl+yh)U==wP_V^4=0y30w;JIojBd-2OgD6wdj|O1G8nOb!4Pa7=5 zh&?#FE2_bU@!j87(dvUw27ew`O(ziT)_$Hy7k%~p`myMmn)fo?r@Lljg1>%2tJp^c z=esE+5%QaZYHPZikku%_uuLi@Jv*CODurKTReI^P#>~k0Q=>0Lkx6HuXH1F&op6;@ zC{DU-(6Ke$;Vg0ML_5M){cvl*Iw`37Tfjt&W6Bpr_VH`ujVR!=U}_2RT2|TIjjpoV zKsu$3AKrar&yDmAIbv&flW{7(_tj8L%~$oU!y0N)-QsrnU1zI-$#)hpt+V4MVV^>; z*ni`{y%gCD13WRnIjX2qzS31-EdO|5h?`J3L9y_u^6k06 zj$>u#lQh=<6QR{cRZS*s6vU+b9iW^<{{Fb(ZmRMM?jHIm(bCYEf@9c!*}+@zgT_^> z3Q<=0tjb5*i(6F{q}=eT@zJj1l+{EiH`Z!=dklDRY&=}JXf`aB3DTrUQ85I_R)dEoB7~xJf&vn% z^nMWOARrQYmm<9dqy+8`=sDl{?)`H=@AKUJU~gt-z588j-nC|)nP<3EEHmml{Q)jI z+E3p&oB0~UetqK{EruieX-`u|9|l0*xZapZw7fXJkB-iWfsT$g7~>#$-`3pK%2NA* z>)m@c_s?S-9p36xYWDudP7Mnt5m#Ef9&3na1&SV|JdR{MctvvBrPRM{bS~qX>uAsY ziiAhyPmsgcue!y@59uDKSO-m%$vRoP@e7=!rpK_`w&%DvXOg1g+_N@h^DKBA-- zHmu1t`z#D~l}@1RYwB@6yoH)mMEqTR{M}_E* zUcGkeI8x2%G;?aq!xC!upl#t4A?alF3p2{(^Eb}0^(U0kl3TCE^#>$3#Z&JNm$nN; zu@I~RadVMVd@Lx|nEJp!DEZIQ&&Vpi(X(MCiD%+ndsKPqC!^(7u=^qDqXdP%( z@Nh8}>vcZ#orjPUwmE&;_l)+7=SM47eSCW3O4_LWT&2%mJ-#x0Gu<$QqhL=9>3um+CjTdlmmesTx;dGM!X7(vP8=yUMxqiN*< zgpqz%EBgwh8)QJOXz!<^;{df{>2wF>>g0T1@{W_M6-_9CAtv@8E^z&^qfze_CGBr( zD}JYn#8NZf)n%fFwAWuogcwlQ>`7!M(;2DK7I)WA>hCFt%Wcr;2$c!Zn~eUy9nsb> znV7s&;uj&h^kh0jR^{4@7ej5ub>4v0hbx$eHBXN~JioZ-+I$$dJ^Iw@m`65!wby5k z;v?dtTx|Vq9bK-ARrY^|zC5VQj;W`wExVAP$Pp9v_E^|z=Vg36*D*d+JihjB$cXsy zKhFXMx7E(P2^qE>K94=Dx-m4ynC}>so3n3r1G(7QxV(A6?%czQv-?u&G%CI6p}=!- zXmppl?*N89FfElC`5qUWBD!5KOX3W^vWG`QDiKB}5+MUaN1ARAP zv7b~@;C4O8DyZ`cjyhN|KhKKeY3%S^To`ZlSf5&m#}pbGw+NJ1$dzunZp1APSK7fn z3S;{vp#HKx6#*T?z+m}yp8j@Tu>{~xK@@G9Q^ptE%5c$8{`8D{0h`@ZVa)JxhbW zX0p{9sds&}0lm3dtXDQYHltUh-HuGJCiK^7!;uLC)GfOreTjQy$xNO#K*zps?5pjP zL)@C47hg%7#LmXz_qg{9cZ!OfmP^(**79o&uU*oe8vHyn^$ya4&XkH3LsDDT&s^EO z^&vYPmwI}&`puo}Htmb<7-!%z^*hx(JKC#6e6$4a=;LJOWMg)2u-tS@f3R7=KVYnK zW^r-uQR%jqUZ~4?0S1sWFsAdJs_IhkT}MGG_BZJrT*<(b-s;Td{*CWz)Pm}MJABYU zIWF2ua>T-Gz3}Yjow7nvU?A`WxaV#N)nfDrzntiGzj66 zgVXElFO%Dfd*q2(Xt8h+Fl&~Pzk1%NAI4W9!2>v21EmA#{*?s)bz+NOF}&XWzJBZ% z8|8s8lco&va$I$_!*a^`Lj}D(3gphyg$C!h&q8II)BQ_E?3TZ$^2{(r*H!h*m)wD- zZ95tCIn*SH*i~fS)TYEWlQj!mBwJbwx>GR)h1VxKr+a06#;D`Pg|DfFMydg30{&B$ zq=?_0G)%)p+&AX|JV}5=87VhJ-!`badS$W8!*HM+g+5Z<{`*Co z9N93irgk;#z2Vt4aWkk#=9wik@;kejp6KPlC~*UO>=%-EhQ{O*AE?f>3UV4dM=3PK zlq%kKH?QewPXy#DM(xHdC!u9}<00j)idqBO=U1;=wO9g}GkgOI$75%T$lg^1ypha$ z*VgUu7n>57ixyTZ7nCZ*XI?vCHk~o16;+WBFty>$IC0(S_Vu~#Zqphit7W|U>&XZy zCkry`r`H$cdbci2wh7-T{HS0|z=_ArT_u)x;AGEnN+d8ib$q=xiOt21U3=x}Yp4(_ z0q^N2K9U58*3;d+osHF*_^~7NG5m~UgUvrnU z)afbP&4uj4TjN>^mu(K*WS6PzW}Osq9*B0yrcV@ED3Uv6_q6bExpbu!r|`{}G6?bk zcTUg#C!@1g5}70IZ_kYuyX*20nsZ#aLz9!Ik{i$E1Eku`#!~vRYdO*RV#!p`%_&M- z%T3Fshl9-vrn8cs%B4L*^Xo*#?8#gqV!j&8Y^6N%R=}Y;RrU&@Jn!5S8VZW&$5JQj zc$wk~xO7{Nn8i2Qykr!vNhn8K#&tJ_uV#{7xh4FRYv!v6t+B~-A)7kx;C(MCdnc8L~PRYIOKY(l+}`FLv8q3)6t7VdiP7` zaEdzGQxUcHFtMk0FVb4>!Wci~yYYR4lg3jvqE*;iNOHvr$-xl;cArudlCv{gI-*B6 zqHCV{_KZtsC>b}uGjq*!IeVTY7j7RrlT^3yjROcIh2=NjHTF+))EvF_wIg~%=V;*d zMIARd8cjiIkvFsL6m;Ewn~U>Birw-sOt0JCw^=T_YD^(dz6zQ>?0k1(R Jyz`-u zYBOgMy`99b!G z@0#YFVjE0KJZhjSZ>DOJps>A$5|F)CMqj)dk|MMzm(1Ngm^(0EXA87%pg1xw17GqH z6?{q=ZU_atBNzCXT2cBO-EK`ecBnl5c6<1^d)^zjO_JJdMMtah0f^q$!r4|Lr8T1K z-qZJI)fhtbEc@JSyRnm*wI&9~cRa{7&1r$bc^BWysE}*wdb+|d_r@ziV=qf{#V0sB zH1=RHwB}<5*N$?i!N7J0Spd;ZV_#d9BQw{~^K~#s=B~Bl)3lo}^A5iU^XE){S~ml= zejzO(fsW-FrxE!KI1p^E<16<4Z3JvLFFD+hB*?3#z#E&Vu1A(RGd%bqKaZv5vu-f9 zk2!U$Io@zc42_IN%m(MY*s#u}DjUkdD0u@hucS_mC75~weWlf{8)Cuz3^Ov)Dexd` zt|xtEw&ynGE>S}zKR0ZfdYe$AqYU%j@S%YcE(sN-Y=qkMT$p*cp2LQiS$deVnUaZ@ z$r9`|%t9lSV~pWsLrH|Mm08g@l0(_K4Wa3&X?q=##HeY|bC0^h+UFikdw*=23mw16 zCZ@{*x*fPXR(ktNm{_Lj-rE!vO|3?e476L)YMY2*zM?WI6jph=iYnK^!{W(Gyju5A z(~hf!R2if6yN&Ua!3edd{;G{j1&2z>84%K_^?g$G_Ta=V8k>X|jHQBwHNk%-?Q|i#H3SB z;Xj|mhwz3r2^#wOTHqP1*f%xxZ|MK{G;FtlkXaUGWy#dkgW&~R2uC6x9aV+vHZ!=)HAQNcF6 zUz*~r87^N+N$1TC+^~LtVrbL(GvjysRjCwysg&1XtY?P zi>1`w?m6Qjlm0oZQZsXe!Y&SN@>k6pX-mh1ibH4pZ92rsT7x-5TQNp=c9#(b zWYm0w#>ivQVTr~_5=uF;$LKbQd$4LZydE+v?I0tzQ&`NgeQesnZ8$}dr%;t4{>`=h z^VPtG6gVE_^&DWW7c61w(y>K(Ff>oTWRAR?$IORr6r|zd|FC@d*L?qBiF$yAv{exp zF(IJl1mS-1A{;BiAEwmP63d+zglA~eorQ%A52+uZ^_h!vXq`I&JG>T?NrZzlUFrvi=PS1_MM!AV25=LPUKKdrsbb zEL8MV!!x`HM@+2J%7XCzAEOaSp-FY`4uZa_7x;lLT1tS7{s#C(djL#`|4#zIhyR(t z>VhyNR%vNjjG;qHbn(-chdR;%lxTWlIWK<*f;JVc@vF>{Z6|r9?d^D_Q^FDX5RPgF zmU_XTf&<^x4RUs?AFfXA9;BmbF}5?8oP<4T6fu54%Nba={&v zlN5`<#qPp@6}GO0uqS;6mWJDFW*Qw;1hwtAXqq{Nd1;x(>0%0S3Quk!ylO-6f*+uz zp(+Ev0@8x~BDC3FxWwk32cg5A*vwB)emP(n~w0AQ><|c@8)qX7V>RKT^R27W-pKV|IT9& zt`5v&4-DimQs~>>7%;T|sXeg!kzWA#MSI{s^?(RgvV%zCB&Z52)NsuDnt$v^P?3hJ z;NmBY-;E+-mRCCQnv^a1igbEA<}7j=ppPqG{^BJ$xc%saD|Vnp^6LsQ5D0H?afF4m11W*Eu6zZH z*X7C_M*X7CIKsl(a|3N$Bl8xGHJ8cLDai@k$izTs&o@S2ac3VT6t~mZC0e>Az^=MGeyz&(zTwDa_ynocv$)%|=+oi3zC6&Uy zvKl*nBT1<2soJsSv4ZMiz*E^8*L$K+{~nGrP-l#P%sN~D5sR)@b@fK=Xyxckwtmer z;6EEtk(pursCt}K)L7Qn)M>`^Z42vFQs&_R^jGk~x#6^^p*7}U9l04up~*m7)X z6m_8Vd!fYfbsH@=*GEf*?)-)ev0ks7JCJ;&yeqtXR!5GfRNVBG0P!Mo%o_)N8Fu0%zt?Azn z(`kl#fH?55{I44fb7yDT1;*d#v>T9mV-4BEb0=xX@E{$|-EaQ1CK4bDLc5eJFB;xr zKLGZF1O7Q?6 diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py index 1cea614f..a22979c4 100644 --- a/orchestration/audit_generator_orchestrator.py +++ b/orchestration/audit_generator_orchestrator.py @@ -7,6 +7,10 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast import openpyxl +from openpyxl.cell.rich_text import CellRichText, TextBlock +from openpyxl.cell.text import InlineFont +from openpyxl.formatting.rule import CellIsRule # type: ignore[reportUnknownVariableType] +from openpyxl.styles import Color, Font from domain.magicplan.models import Door, Plan, Room, Window from infrastructure.postgres.uploaded_file_table import ( @@ -23,6 +27,26 @@ _TEMPLATE_PATH = Path(__file__).parent.parent / "applications" / "audit_generato _SHEET_NAME = "D1 Ventilation" _DATA_START_ROW = 6 _MAX_ROWS = 50 +_Y_CF_RANGE = f"Y{_DATA_START_ROW}:Y{_DATA_START_ROW + _MAX_ROWS - 1}" +_Y_THRESHOLD = 7600 +_Y_HEADER = CellRichText( + TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), "Area (mm2)\n"), + TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="FF0000"), rFont="Aptos Narrow"), "<"), + TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), " 7600 "), + TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="196B24"), rFont="Aptos Narrow"), "<"), +) + + +def _apply_column_y_formatting(sheet: Any) -> None: + sheet.conditional_formatting.add( + _Y_CF_RANGE, + CellIsRule(operator="lessThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="FF0000"))), + ) + sheet.conditional_formatting.add( + _Y_CF_RANGE, + CellIsRule(operator="greaterThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="196B24"))), + ) + sheet["Y3"] = _Y_HEADER def _write_cell(sheet: Any, row: int, col: str, value: Any) -> None: @@ -75,6 +99,8 @@ def _populate_sheet(sheet: Any, plan: Plan) -> None: _write_cell(sheet, row, "X", vent.undercut_mm if vent else 0) # Y = formula =W*X — do not write + _apply_column_y_formatting(sheet) + def _serialise_workbook(wb: Any) -> bytes: buf = BytesIO() From b9eb23f6df0c248a2c833477ace872be6aa66dca Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 13:52:39 +0000 Subject: [PATCH 14/18] allow write to real s3 when running locally --- scripts/run_audit_generator_local.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/run_audit_generator_local.py b/scripts/run_audit_generator_local.py index 748e3f89..ab0215c8 100644 --- a/scripts/run_audit_generator_local.py +++ b/scripts/run_audit_generator_local.py @@ -1,18 +1,21 @@ """ -Run audit_generator locally. Writes XLSX to ./local_output/ instead of S3. +Run audit_generator locally. Usage: cd /workspaces/model - python scripts/run_audit_generator_local.py + python scripts/run_audit_generator_local.py [] + +Prompts for deal ID and S3 destination (local file or real S3) if not supplied. """ from __future__ import annotations import os import sys -from io import BytesIO from pathlib import Path -from typing import Any +from typing import Any, Union + +import boto3 # Load .env before importing infra modules from dotenv import load_dotenv @@ -21,6 +24,7 @@ load_dotenv(Path(__file__).parent.parent / "backend" / ".env") from infrastructure.postgres.config import PostgresConfig from infrastructure.postgres.engine import make_engine, make_session +from infrastructure.s3.s3_client import S3Client from orchestration.audit_generator_orchestrator import AuditGeneratorOrchestrator from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork @@ -46,9 +50,19 @@ class _LocalS3Client: return str(dest) +def _make_s3_client() -> Union[S3Client, "_LocalS3Client"]: + use_real = input("Use real S3? [y/N]: ").strip().lower() == "y" + if use_real: + bucket = "retrofit-energy-assessments-dev" + boto3_client: Any = boto3.client + return S3Client(boto_s3_client=boto3_client("s3"), bucket=bucket) + output_dir = Path(__file__).parent.parent / "local_output" + return _LocalS3Client(output_dir) + + def main() -> None: deal_id = sys.argv[1] if len(sys.argv) > 1 else input("hubspot_deal_id: ").strip() - output_dir = Path(__file__).parent.parent / "local_output" + s3_client = _make_s3_client() engine = make_engine(PostgresConfig.from_env(os.environ)) @@ -60,7 +74,7 @@ def main() -> None: AuditGeneratorOrchestrator( hubspot_deal_id=deal_id, - s3_client=_LocalS3Client(output_dir), # type: ignore[arg-type] + s3_client=s3_client, # type: ignore[arg-type] uow_factory=uow_factory, ).run() From 163703285f849d7a5acedaf79783eb9268e6b0a4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 14:15:52 +0000 Subject: [PATCH 15/18] deployment --- .github/workflows/deploy_terraform.yml | 36 +++++++++++++ .github/workflows/lambda_smoke_tests.yml | 10 ++++ .../audit_generator/handler/Dockerfile | 17 ++++++ .../audit_generator/handler/requirements.txt | 7 +++ .../terraform/lambda/audit_generator/main.tf | 45 ++++++++++++++++ .../lambda/audit_generator/outputs.tf | 9 ++++ .../lambda/audit_generator/provider.tf | 16 ++++++ .../lambda/audit_generator/variables.tf | 52 +++++++++++++++++++ deployment/terraform/shared/main.tf | 14 +++++ 9 files changed, 206 insertions(+) create mode 100644 applications/audit_generator/handler/Dockerfile create mode 100644 applications/audit_generator/handler/requirements.txt create mode 100644 deployment/terraform/lambda/audit_generator/main.tf create mode 100644 deployment/terraform/lambda/audit_generator/outputs.tf create mode 100644 deployment/terraform/lambda/audit_generator/provider.tf create mode 100644 deployment/terraform/lambda/audit_generator/variables.tf diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index e1e9b3c5..338ef11d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -661,6 +661,42 @@ jobs: TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }} TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }} + # ============================================================ + # Build Audit Generator image + # ============================================================ + audit_generator_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: audit-generator-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: applications/audit_generator/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Audit Generator Lambda + # ============================================================ + audit_generator_lambda: + needs: [audit_generator_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: audit_generator + lambda_path: deployment/terraform/lambda/audit_generator + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: audit-generator-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.audit_generator_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + # ============================================================ # Deploy Hubspot ETL Lambda # ============================================================ diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index 329a1319..1830fcd9 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -123,6 +123,16 @@ jobs: build_context: . service_name: magic-plan + # ============================================================ + # Audit Generator + # ============================================================ + audit_generator_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: applications/audit_generator/handler/Dockerfile + build_context: . + service_name: audit-generator + # ============================================================ # HubSpot Scraper # ============================================================ diff --git a/applications/audit_generator/handler/Dockerfile b/applications/audit_generator/handler/Dockerfile new file mode 100644 index 00000000..e70b00a3 --- /dev/null +++ b/applications/audit_generator/handler/Dockerfile @@ -0,0 +1,17 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/audit_generator/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utilities/ utilities/ +COPY backend/ backend/ +COPY applications/ applications/ +COPY domain/ domain/ +COPY datatypes/ datatypes/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY infrastructure/ infrastructure/ + +CMD ["applications.audit_generator.handler.handler"] diff --git a/applications/audit_generator/handler/requirements.txt b/applications/audit_generator/handler/requirements.txt new file mode 100644 index 00000000..bf2c1bff --- /dev/null +++ b/applications/audit_generator/handler/requirements.txt @@ -0,0 +1,7 @@ +awslambdaric +sqlalchemy==2.0.36 +sqlmodel +psycopg2-binary==2.9.10 +pydantic-settings==2.6.0 +boto3==1.35.44 +openpyxl diff --git a/deployment/terraform/lambda/audit_generator/main.tf b/deployment/terraform/lambda/audit_generator/main.tf new file mode 100644 index 00000000..64e36aa4 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/main.tf @@ -0,0 +1,45 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +resource "aws_iam_role_policy_attachment" "audit_generator_s3_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "audit_generator" + stage = var.stage + + image_uri = local.image_uri + + maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_HOST = var.db_host + POSTGRES_DATABASE = var.db_name + POSTGRES_PORT = var.db_port + } +} diff --git a/deployment/terraform/lambda/audit_generator/outputs.tf b/deployment/terraform/lambda/audit_generator/outputs.tf new file mode 100644 index 00000000..04609ca6 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/outputs.tf @@ -0,0 +1,9 @@ +output "audit_generator_queue_url" { + value = module.lambda.queue_url + description = "URL of the Audit Generator SQS queue" +} + +output "audit_generator_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the Audit Generator SQS queue" +} diff --git a/deployment/terraform/lambda/audit_generator/provider.tf b/deployment/terraform/lambda/audit_generator/provider.tf new file mode 100644 index 00000000..3793fefd --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "audit-generator-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/terraform/lambda/audit_generator/variables.tf b/deployment/terraform/lambda/audit_generator/variables.tf new file mode 100644 index 00000000..4642d811 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/variables.tf @@ -0,0 +1,52 @@ +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index 0bd7a966..7ca116e7 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -830,3 +830,17 @@ module "magic_plan_client_registry" { stage = var.stage } +################################################ +# Audit Generator – Lambda +################################################ +module "audit_generator_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "audit-generator-terraform-state" +} + +module "audit_generator_registry" { + source = "../modules/container_registry" + name = "audit-generator" + stage = var.stage +} + From 48a413a5e297f4622dc134b161e0910fcd911ce3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 14:25:29 +0000 Subject: [PATCH 16/18] correct imports in postgres repo --- .../magic_plan/magic_plan_postgres_repository.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/repositories/magic_plan/magic_plan_postgres_repository.py b/repositories/magic_plan/magic_plan_postgres_repository.py index 1cd77a32..f8ff123c 100644 --- a/repositories/magic_plan/magic_plan_postgres_repository.py +++ b/repositories/magic_plan/magic_plan_postgres_repository.py @@ -6,7 +6,15 @@ from sqlalchemy import delete, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlmodel import Session, col -from domain.magicplan.models import Floor, Plan +from domain.magicplan.models import ( + Door, + DoorVentilation, + Floor, + Plan, + Room, + Window, + WindowVentilation, +) from infrastructure.postgres.magic_plan_tables import ( MagicPlanDoorModel, MagicPlanDoorVentilationModel, From 236f33c25fb3aa58585b2bd004d7a092c42b8144 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 14:43:24 +0000 Subject: [PATCH 17/18] move spreadsheet population logic to domain --- .../extract-populate-sheet-to-domain.md | 68 ++++++++++++++ domain/magicplan/ventilation_audit.py | 86 +++++++++++++++++ orchestration/audit_generator_orchestrator.py | 83 +--------------- .../magicplan/test_ventilation_audit.py | 94 +++++++++++++++++++ .../test_audit_generator_orchestrator.py | 15 --- 5 files changed, 250 insertions(+), 96 deletions(-) create mode 100644 docs/backlog/extract-populate-sheet-to-domain.md create mode 100644 domain/magicplan/ventilation_audit.py create mode 100644 tests/domain/magicplan/test_ventilation_audit.py diff --git a/docs/backlog/extract-populate-sheet-to-domain.md b/docs/backlog/extract-populate-sheet-to-domain.md new file mode 100644 index 00000000..59474895 --- /dev/null +++ b/docs/backlog/extract-populate-sheet-to-domain.md @@ -0,0 +1,68 @@ +# PRD: Extract ventilation audit sheet population into the magicplan domain + +**Status:** Backlog + +--- + +## Problem Statement + +The logic that maps a `Plan` into spreadsheet cells — which column receives `pct_openable / 100`, which rows are rooms vs windows vs doors, what the 50-row capacity limit is, how column Y conditional formatting is applied — currently lives inside the orchestrator. Developers reading `AuditGeneratorOrchestrator` have to wade through cell-writing details to understand the orchestration flow, and there is no way to test the sheet-population rules in isolation without invoking the full orchestrator (which requires a mocked UoW, mocked S3, and the real XLSX template file). + +## Solution + +Move all sheet-population logic into the magicplan domain as a dedicated module (`ventilation_audit`), exposing a single public function `populate_sheet(sheet, plan)`. The orchestrator delegates to this function and retains only its infrastructure responsibilities: loading the template, serialising the workbook, uploading to S3, and persisting metadata. + +This makes the mapping rules directly testable against a plain `openpyxl` sheet with no orchestration overhead, and keeps the orchestrator focused on coordination rather than domain rules. + +## User Stories + +1. As a developer debugging a malformed audit spreadsheet, I want the cell-mapping rules to live in the domain so that I can locate the logic without reading through orchestration code. +2. As a developer writing a test for ventilation audit content, I want to call `populate_sheet` directly with a synthetic `Plan` and a blank sheet so that I can assert cell values without mocking S3 or a unit of work. +3. As a developer adding a new opening type or ventilation field, I want the affected mapping logic to be co-located with the `Plan` domain models so that the change is easy to find and the impact is obvious. +4. As a developer reading the orchestrator, I want the `run()` method to read as a sequence of high-level steps (fetch → populate → serialise → upload → persist) with no cell-writing detail so that the orchestration intent is immediately clear. +5. As a developer running the test suite, I want the 50-row overflow validation to be covered by a domain-level test so that regressions in that constraint are caught without running the full orchestrator. +6. As a developer extending the audit template to a second sheet, I want the sheet-population contract to be a clearly bounded function so that I can add a second `populate_*` function in the same module without touching the orchestrator. + +## Implementation Decisions + +- **New module `domain/magicplan/ventilation_audit.py`** contains the public function `populate_sheet(sheet, plan)` and all private helpers (`_write_cell`, `_apply_column_y_formatting`) and constants (`_DATA_START_ROW`, `_MAX_ROWS`, `_Y_CF_RANGE`, `_Y_THRESHOLD`, `_Y_HEADER`). These are moved verbatim from the orchestrator — no logic changes. + +- **`populate_sheet` is the sole public surface.** Helpers remain private to the module. This follows the existing `mapper.py` pattern (stateless module-level functions, no class wrapper). + +- **The orchestrator imports `populate_sheet`** and replaces its `_populate_sheet(sheet, plan)` call. All `openpyxl.cell.rich_text`, `openpyxl.cell.text`, `openpyxl.formatting.rule`, and `openpyxl.styles` imports move with the logic. `openpyxl.load_workbook` stays — loading the template is an infrastructure step. + +- **`_serialise_workbook` stays in the orchestrator** — converting a workbook to bytes is a serialisation step, not domain logic. + +- **No interface change to the orchestrator's public API** — `AuditGeneratorOrchestrator.__init__` and `run()` signatures are unchanged. + +## Testing Decisions + +Good tests for `populate_sheet` assert observable outputs (cell values, conditional formatting rule count) given a controlled `Plan` input. They do not assert on internal call sequences or private helper invocations. + +Tests should use a fresh `openpyxl.Workbook().active` sheet — no template file needed, which keeps them fast and dependency-free. + +Modules to test (new file: `tests/domain/magicplan/test_ventilation_audit.py`): + +| Scenario | Assertion | +|---|---| +| Rooms written correctly | Col B = room name, col D = area_m2, starting at `_DATA_START_ROW` | +| Windows written correctly | Cols G–I, K–M, Q–R populated; pct_openable divided by 100 | +| Windows with null ventilation | Ventilation columns default to 0 | +| Doors written correctly | Cols V–X populated with room name, width_mm, undercut_mm | +| Room overflow | > 50 rooms raises `ValueError` | +| Window overflow | > 50 windows raises `ValueError` | +| Door overflow | > 50 doors raises `ValueError` | +| Column Y formatting applied | Sheet has two conditional formatting rules after `populate_sheet` | + +Prior art: `tests/orchestration/audit_generator/test_audit_generator_orchestrator.py` shows the `_make_plan` / `_make_window` / `_make_door` fixture pattern to reuse. The existing orchestrator tests need no changes. + +## Out of Scope + +- Changes to the spreadsheet template or column layout. +- Support for plans with more than 50 rooms, windows, or doors (the 50-row limit is a template constraint, not lifted here). +- Extracting `_serialise_workbook` or template-loading into the domain. +- Any changes to the `AuditGeneratorOrchestrator` public API or the Lambda entry point. + +## Further Notes + +The orchestrator test suite already provides integration-level coverage (S3 call order, `UploadedFile` enums, error paths). This refactor adds the missing unit-level coverage for the mapping rules, which are currently exercised only incidentally via the happy-path orchestrator tests. diff --git a/domain/magicplan/ventilation_audit.py b/domain/magicplan/ventilation_audit.py new file mode 100644 index 00000000..36214412 --- /dev/null +++ b/domain/magicplan/ventilation_audit.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Any + +from openpyxl.cell.rich_text import CellRichText, TextBlock +from openpyxl.cell.text import InlineFont +from openpyxl.formatting.rule import CellIsRule # type: ignore[reportUnknownVariableType] +from openpyxl.styles import Color, Font + +from domain.magicplan.models import Door, Plan, Room, Window + +_DATA_START_ROW = 6 +_MAX_ROWS = 50 +_Y_CF_RANGE = f"Y{_DATA_START_ROW}:Y{_DATA_START_ROW + _MAX_ROWS - 1}" +_Y_THRESHOLD = 7600 +_Y_HEADER = CellRichText( + TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), "Area (mm2)\n"), + TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="FF0000"), rFont="Aptos Narrow"), "<"), + TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), " 7600 "), + TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="196B24"), rFont="Aptos Narrow"), "<"), +) + + +def _apply_column_y_formatting(sheet: Any) -> None: + sheet.conditional_formatting.add( + _Y_CF_RANGE, + CellIsRule(operator="lessThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="FF0000"))), + ) + sheet.conditional_formatting.add( + _Y_CF_RANGE, + CellIsRule(operator="greaterThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="196B24"))), + ) + sheet["Y3"] = _Y_HEADER + + +def _write_cell(sheet: Any, row: int, col: str, value: Any) -> None: + sheet[f"{col}{row}"] = value + + +def populate_sheet(sheet: Any, plan: Plan) -> None: + rooms: list[Room] = [room for floor in plan.floors for room in floor.rooms] + windows: list[tuple[str, Window]] = [ + (room.name, w) for room in rooms for w in room.windows + ] + doors: list[tuple[str, Door]] = [ + (room.name, d) for room in rooms for d in room.doors + ] + + if len(rooms) > _MAX_ROWS: + raise ValueError(f"Room series exceeds {_MAX_ROWS} rows ({len(rooms)} rooms)") + if len(windows) > _MAX_ROWS: + raise ValueError(f"Window series exceeds {_MAX_ROWS} rows ({len(windows)} windows)") + if len(doors) > _MAX_ROWS: + raise ValueError(f"Door series exceeds {_MAX_ROWS} rows ({len(doors)} doors)") + + for i, room in enumerate(rooms): + row = _DATA_START_ROW + i + _write_cell(sheet, row, "B", room.name) + _write_cell(sheet, row, "D", room.area_m2) + + for i, (room_name, window) in enumerate(windows): + row = _DATA_START_ROW + i + vent = window.ventilation + _write_cell(sheet, row, "G", room_name) + _write_cell(sheet, row, "H", window.width_m) + _write_cell(sheet, row, "I", window.height_m) + # J = formula =H*I — do not write + _write_cell(sheet, row, "K", vent.opening_type if vent else 0) + _write_cell(sheet, row, "L", vent.num_openings if vent else 0) + pct = vent.pct_openable if vent else None + _write_cell(sheet, row, "M", (pct / 100) if pct is not None else 0) + # N = formula =J*M — do not write + # O, P = blank (visual check by auditor) + _write_cell(sheet, row, "Q", vent.trickle_vent_area_mm2 if vent else 0) + _write_cell(sheet, row, "R", vent.num_trickle_vents if vent else 0) + # S = formula =Q*R — do not write + + for i, (room_name, door) in enumerate(doors): + row = _DATA_START_ROW + i + vent = door.ventilation + _write_cell(sheet, row, "V", room_name) + _write_cell(sheet, row, "W", door.width_mm) + _write_cell(sheet, row, "X", vent.undercut_mm if vent else 0) + # Y = formula =W*X — do not write + + _apply_column_y_formatting(sheet) diff --git a/orchestration/audit_generator_orchestrator.py b/orchestration/audit_generator_orchestrator.py index a22979c4..3e436348 100644 --- a/orchestration/audit_generator_orchestrator.py +++ b/orchestration/audit_generator_orchestrator.py @@ -7,12 +7,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast import openpyxl -from openpyxl.cell.rich_text import CellRichText, TextBlock -from openpyxl.cell.text import InlineFont -from openpyxl.formatting.rule import CellIsRule # type: ignore[reportUnknownVariableType] -from openpyxl.styles import Color, Font -from domain.magicplan.models import Door, Plan, Room, Window +from domain.magicplan.ventilation_audit import populate_sheet from infrastructure.postgres.uploaded_file_table import ( FileSourceEnum, FileTypeEnum, @@ -25,81 +21,6 @@ if TYPE_CHECKING: _TEMPLATE_PATH = Path(__file__).parent.parent / "applications" / "audit_generator" / "d1_ventilation_template.xlsx" _SHEET_NAME = "D1 Ventilation" -_DATA_START_ROW = 6 -_MAX_ROWS = 50 -_Y_CF_RANGE = f"Y{_DATA_START_ROW}:Y{_DATA_START_ROW + _MAX_ROWS - 1}" -_Y_THRESHOLD = 7600 -_Y_HEADER = CellRichText( - TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), "Area (mm2)\n"), - TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="FF0000"), rFont="Aptos Narrow"), "<"), - TextBlock(InlineFont(b=True, sz=11, rFont="Aptos Narrow"), " 7600 "), - TextBlock(InlineFont(b=True, sz=11, color=Color(rgb="196B24"), rFont="Aptos Narrow"), "<"), -) - - -def _apply_column_y_formatting(sheet: Any) -> None: - sheet.conditional_formatting.add( - _Y_CF_RANGE, - CellIsRule(operator="lessThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="FF0000"))), - ) - sheet.conditional_formatting.add( - _Y_CF_RANGE, - CellIsRule(operator="greaterThan", formula=[str(_Y_THRESHOLD)], font=Font(color=Color(rgb="196B24"))), - ) - sheet["Y3"] = _Y_HEADER - - -def _write_cell(sheet: Any, row: int, col: str, value: Any) -> None: - sheet[f"{col}{row}"] = value - - -def _populate_sheet(sheet: Any, plan: Plan) -> None: - rooms: list[Room] = [room for floor in plan.floors for room in floor.rooms] - windows: list[tuple[str, Window]] = [ - (room.name, w) for room in rooms for w in room.windows - ] - doors: list[tuple[str, Door]] = [ - (room.name, d) for room in rooms for d in room.doors - ] - - if len(rooms) > _MAX_ROWS: - raise ValueError(f"Room series exceeds {_MAX_ROWS} rows ({len(rooms)} rooms)") - if len(windows) > _MAX_ROWS: - raise ValueError(f"Window series exceeds {_MAX_ROWS} rows ({len(windows)} windows)") - if len(doors) > _MAX_ROWS: - raise ValueError(f"Door series exceeds {_MAX_ROWS} rows ({len(doors)} doors)") - - for i, room in enumerate(rooms): - row = _DATA_START_ROW + i - _write_cell(sheet, row, "B", room.name) - _write_cell(sheet, row, "D", room.area_m2) - - for i, (room_name, window) in enumerate(windows): - row = _DATA_START_ROW + i - vent = window.ventilation - _write_cell(sheet, row, "G", room_name) - _write_cell(sheet, row, "H", window.width_m) - _write_cell(sheet, row, "I", window.height_m) - # J = formula =H*I — do not write - _write_cell(sheet, row, "K", vent.opening_type if vent else 0) - _write_cell(sheet, row, "L", vent.num_openings if vent else 0) - pct = vent.pct_openable if vent else None - _write_cell(sheet, row, "M", (pct / 100) if pct is not None else 0) - # N = formula =J*M — do not write - # O, P = blank (visual check by auditor) - _write_cell(sheet, row, "Q", vent.trickle_vent_area_mm2 if vent else 0) - _write_cell(sheet, row, "R", vent.num_trickle_vents if vent else 0) - # S = formula =Q*R — do not write - - for i, (room_name, door) in enumerate(doors): - row = _DATA_START_ROW + i - vent = door.ventilation - _write_cell(sheet, row, "V", room_name) - _write_cell(sheet, row, "W", door.width_mm) - _write_cell(sheet, row, "X", vent.undercut_mm if vent else 0) - # Y = formula =W*X — do not write - - _apply_column_y_formatting(sheet) def _serialise_workbook(wb: Any) -> bytes: @@ -138,7 +59,7 @@ class AuditGeneratorOrchestrator: wb = openpyxl.load_workbook(_TEMPLATE_PATH) sheet = wb[_SHEET_NAME] - _populate_sheet(sheet, plan) + populate_sheet(sheet, plan) xlsx_bytes = _serialise_workbook(wb) s3_key = ( diff --git a/tests/domain/magicplan/test_ventilation_audit.py b/tests/domain/magicplan/test_ventilation_audit.py new file mode 100644 index 00000000..eb7ba7cc --- /dev/null +++ b/tests/domain/magicplan/test_ventilation_audit.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import openpyxl +import pytest + +from domain.magicplan.models import ( + Door, + DoorVentilation, + Floor, + Plan, + Room, + Window, + WindowVentilation, +) +from domain.magicplan.ventilation_audit import populate_sheet + + +def _make_window(with_ventilation: bool = True) -> Window: + vent = ( + WindowVentilation( + opening_type="Hinged", + num_openings=1, + pct_openable=50, + trickle_vent_area_mm2=1000, + num_trickle_vents=2, + ) + if with_ventilation + else None + ) + return Window(width_m=1.0, height_m=1.2, area_m2=1.2, ventilation=vent) + + +def _make_door(with_ventilation: bool = True) -> Door: + vent = DoorVentilation(undercut_mm=10.0) if with_ventilation else None + return Door(width_mm=800.0, height_mm=2000.0, ventilation=vent) + + +def _make_plan( + num_rooms: int = 1, + num_windows_per_room: int = 1, + num_doors_per_room: int = 1, +) -> Plan: + rooms = [ + Room( + name=f"Room {i}", + width_m=3.0, + length_m=4.0, + area_m2=12.0, + windows=[_make_window() for _ in range(num_windows_per_room)], + doors=[_make_door() for _ in range(num_doors_per_room)], + ) + for i in range(num_rooms) + ] + return Plan( + uid="test-uid", + name="Test Plan", + address="1 Test St", + postcode="TE1 1ST", + floors=[Floor(level=0, name="Ground", rooms=rooms)], + ) + + +def _blank_sheet() -> object: + return openpyxl.Workbook().active + + +def test_raises_when_rooms_exceed_50() -> None: + # Arrange + plan = _make_plan(num_rooms=51, num_windows_per_room=0, num_doors_per_room=0) + sheet = _blank_sheet() + + # Act / Assert + with pytest.raises(ValueError, match="50"): + populate_sheet(sheet, plan) + + +def test_raises_when_windows_exceed_50() -> None: + # Arrange + plan = _make_plan(num_rooms=1, num_windows_per_room=51, num_doors_per_room=0) + sheet = _blank_sheet() + + # Act / Assert + with pytest.raises(ValueError, match="50"): + populate_sheet(sheet, plan) + + +def test_raises_when_doors_exceed_50() -> None: + # Arrange + plan = _make_plan(num_rooms=1, num_windows_per_room=0, num_doors_per_room=51) + sheet = _blank_sheet() + + # Act / Assert + with pytest.raises(ValueError, match="50"): + populate_sheet(sheet, plan) diff --git a/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py b/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py index 18445fe5..2d9d706c 100644 --- a/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py +++ b/tests/orchestration/audit_generator/test_audit_generator_orchestrator.py @@ -202,18 +202,3 @@ def test_commits_after_s3_upload() -> None: # Assert — S3 upload happens before DB commit assert call_order == ["s3_upload", "commit"] - -# --- 50-row limit --- - - -def test_raises_when_series_exceeds_50_rows() -> None: - # Arrange — 51 windows, which exceeds the template limit - plan = _make_plan(num_rooms=1, num_windows_per_room=51) - mock_uow, mock_uow_factory = _make_mock_uow( - uploaded_file_row=_make_uploaded_file_row(), plan=plan - ) - orch = _make_orchestrator(uow_factory=mock_uow_factory) - - # Act / Assert - with pytest.raises(ValueError, match="50"): - orch.run() From dcd5204b544cb1e1b7b958bf3aeb01ea664c1e36 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 9 Jun 2026 15:04:12 +0000 Subject: [PATCH 18/18] put db engine construction inside handler to avoid import errors in test --- applications/audit_generator/handler.py | 7 +++---- .../audit_generator/test_audit_generator_handler.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/applications/audit_generator/handler.py b/applications/audit_generator/handler.py index b560f808..533f5d91 100644 --- a/applications/audit_generator/handler.py +++ b/applications/audit_generator/handler.py @@ -15,9 +15,6 @@ from orchestration.audit_generator_orchestrator import AuditGeneratorOrchestrato from orchestration.audit_generator_unit_of_work import AuditGeneratorUnitOfWork from utilities.aws_lambda.subtask_handler import subtask_handler -_engine = make_engine(PostgresConfig.from_env(os.environ)) - - @subtask_handler() def handler(body: dict[str, Any], context: Any) -> None: trigger = AuditGeneratorTriggerRequest.model_validate(body) @@ -27,8 +24,10 @@ def handler(body: dict[str, Any], context: Any) -> None: bucket = os.environ["S3_BUCKET_NAME"] s3_client = S3Client(boto_s3_client=boto_s3, bucket=bucket) + engine = make_engine(PostgresConfig.from_env(os.environ)) + def session_factory() -> Any: - return make_session(_engine) + return make_session(engine) def uow_factory() -> AuditGeneratorUnitOfWork: return AuditGeneratorUnitOfWork(session_factory) diff --git a/tests/applications/audit_generator/test_audit_generator_handler.py b/tests/applications/audit_generator/test_audit_generator_handler.py index f9bc74b6..b04a6f8a 100644 --- a/tests/applications/audit_generator/test_audit_generator_handler.py +++ b/tests/applications/audit_generator/test_audit_generator_handler.py @@ -11,9 +11,8 @@ from applications.audit_generator.handler import handler _ENV = { "DATABASE_URL": "postgresql+psycopg://user:pass@localhost/db", "S3_BUCKET_NAME": "test-bucket", - # PostgresConfig.from_env also reads these individual vars; DATABASE_URL is - # used directly when constructing the engine in the handler module scope, so - # we patch make_engine instead of the env. + # Tests patch PostgresConfig and make_engine to avoid needing the individual + # POSTGRES_* vars that PostgresConfig.from_env would otherwise require. } _VALID_BODY: dict[str, Any] = { @@ -50,6 +49,8 @@ def test_handler_passes_hubspot_deal_id_from_body_to_orchestrator() -> None: # Act with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.PostgresConfig"), \ + patch("applications.audit_generator.handler.make_engine"), \ patch("applications.audit_generator.handler.S3Client") as MockS3, \ patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch) as MockOrch: MockS3.return_value = MagicMock() @@ -67,6 +68,8 @@ def test_handler_passes_bucket_from_env_to_s3_client() -> None: # Act with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.PostgresConfig"), \ + patch("applications.audit_generator.handler.make_engine"), \ patch("applications.audit_generator.handler.S3Client") as MockS3, \ patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch): _call(_VALID_BODY) @@ -86,6 +89,8 @@ def test_handler_returns_none_on_success() -> None: # Act with patch("applications.audit_generator.handler.os.environ", _ENV), \ + patch("applications.audit_generator.handler.PostgresConfig"), \ + patch("applications.audit_generator.handler.make_engine"), \ patch("applications.audit_generator.handler.S3Client"), \ patch("applications.audit_generator.handler.AuditGeneratorOrchestrator", return_value=mock_orch): result = _call(_VALID_BODY)