From 01e8b8c289c891af7092e59376bb91617dadf7a8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:08:26 +0000 Subject: [PATCH 01/46] =?UTF-8?q?Record=20the=20Scenario=20Export=20design?= =?UTF-8?q?=20as=20ADR-0065=20with=20glossary=20terms=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- CONTEXT.md | 24 ++- docs/adr/0065-ara-scenario-export.md | 210 +++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 docs/adr/0065-ara-scenario-export.md diff --git a/CONTEXT.md b/CONTEXT.md index 047087e97..5cc6c539a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -105,12 +105,34 @@ _Avoid_: document category, file kind, doc class **Download Package**: The single ZIP archive a **Bulk Document Download** produces: one folder per property (named by human-readable address, enriched from the hubspot deals data, with `landlord_property_id` appended for uniqueness), each folder holding the latest file of each **Document Type** held for that property. Built best-effort — properties/documents that don't resolve are skipped and listed in the run's `sub_task.outputs`, not failed (ADR-0060). Streamed to a dedicated `DOCUMENT_EXPORTS_BUCKET` (never held whole in memory — a package can be several GB); delivered as a 60-minute presigned URL. -_Avoid_: export (that is the plan/scenario XLSX export), bundle, archive (ambiguous), zip (the format, not the concept) +_Avoid_: export (that is the **Scenario Export**), bundle, archive (ambiguous), zip (the format, not the concept) **Bulk Document Download**: The feature/job that assembles a **Download Package** for a chosen set of properties (the union of one or more whole HubSpot projects, selected by `project_code`, and a hand-picked `landlord_property_id` list) and emails the requester a link. Runs on the app-owned-task + attach-mode lane (ADR-0055): the FastAPI route resolves the selection to a distinct `landlord_property_id` set — project codes are expanded via `hubspot_deal_data` (which holds the project↔property grain), hand-picked ids are taken as given — pins that set and the recipient email onto the `sub_task`, and the `applications/bulk_document_download` Lambda builds the package, writes the URL to `sub_task.outputs`, and emails it (ADR-0059/0060). The Lambda matches files by `hubspot_deal_id`, bridging each selected `landlord_property_id` through `hubspot_deal_data` (`uploaded_files.landlord_property_id` is unpopulated in practice, so the property identity comes from the deal bridge); coverage is limited to deal-id-linked sources. Selection is capped by property count at the trigger. _Avoid_: document export, bulk export, file dump +**Scenario Export** (canonical short form: **Export**): +The single branded `.xlsx` a user pulls from the front end to get a portfolio's +**modelling results** — property information plus each Property's **Plan** under +one or more **Scenarios** — with a link emailed to the requester. It is +**scenario-scoped**: the rows for a Scenario are the Properties that have a +**Plan** under it, so a Property with no Plan for that Scenario is absent (model +A, ADR-0065); a filter or a hand-picked id list only narrows that set. One +workbook holds **one sheet per selected Scenario** (`scenario_ids`), the same +filtered Property set rendered per Scenario — so sheets may legitimately differ +in row count. Property selection reuses the **Modelling Run**'s filter contract +(`PropertyGroupFilters` + `resolve_filtered_property_ids`, ADR-0056); an explicit +`property_ids` list is an **override** of the filters, not a co-filter. Built +**best-effort** from **persisted** data only (no live EPC calls), reading each +Property's **Effective EPC** (so `property_overrides` win over the lodged cert), +against a frozen 21-measure column contract. Runs on the app-owned-task + +attach-mode lane and is delivered by presigned URL + email exactly like a +**Download Package** (ADR-0055/0059), but its payload is plan/scenario data, not +survey documents. Distinct from a **Download Package** (the ZIP of survey +documents) and from a **Modelling Run** (which *produces* the Plans an Export +reads). +_Avoid_: data export / bulk export (ambiguous), download (that is the **Download Package**), report, dump, plan export (it is scenario-scoped, not per-Plan) + ### Source data **Site Notes**: diff --git a/docs/adr/0065-ara-scenario-export.md b/docs/adr/0065-ara-scenario-export.md new file mode 100644 index 000000000..7cebda333 --- /dev/null +++ b/docs/adr/0065-ara-scenario-export.md @@ -0,0 +1,210 @@ +--- +status: accepted (builds on ADR-0055, ADR-0056, ADR-0059, ADR-0060) +--- + +# ARA Scenario Export builds one branded workbook per request, sheet-per-scenario + +Users need to pull a portfolio's modelling results — property information plus +each Property's **Plan** under one or more **Scenarios** — out of the product as +a single spreadsheet, initiated from the front end and delivered as a link +emailed to the requester. This is the **Export** the glossary already reserves +the word for ("the plan/scenario XLSX export"), distinct from a **Download +Package** (the ZIP of survey documents, ADR-0060). + +The data layer already half-exists as `scripts/data_exports.py` (Jun-te Kim): a +CLI over the new DDD schema that reads `property` / `scenario` / `plan` / +`recommendation` / `material`, pivots measures onto a **stable 21-measure column +contract** (`EXPECTED_MEASURE_COLUMNS`), and emits one sheet per scenario. But +as written it (a) fetches EPC descriptive fields **live** from the gov API per +property (the retired `property_details_epc` table's replacement), (b) has **no +filters** and auto-derives every scenario with plans, (c) lives in `scripts/` +outside the DDD layout, and (d) has no styling, S3, or email. None of those +survive a front-end feature that must scope by scenario + filters and scale to +**tens of thousands** of properties inside a Lambda. + +The trigger/email/large-selection plumbing is already solved by +**Bulk Document Download** (ADR-0055/0059/0060) and is reused wholesale. + +## Decision + +**A request produces exactly one branded `.xlsx` Export: one sheet per selected +Scenario, over a filtered, scenario-scoped Property set, built from persisted +data, on the app-owned-task + attach-mode lane (ADR-0055), delivered by +presigned URL + email (ADR-0059).** + +- **Scenario-scoped selection (model A).** The unit of scope is *a Scenario's + results*. The base set for each sheet is the Properties that have a **Plan** + under that Scenario; filters and any hand-picked ids only **narrow** it. A + Property that filters in but has **no Plan for that Scenario drops out** of + that sheet — there is nothing to export for it. Scenario membership is applied + in the data layer (the `plan` join), not in the filter resolver. + +- **Sheet-per-Scenario, one workbook (A2).** The recipe carries + `scenario_ids: list[int]`; the **same** filtered Property set is rendered once + per Scenario, each on its own sheet. Under model A the sheets may legitimately + have **different row counts** (a Property with a Plan under Scenario X but not + Y appears on the X sheet only). Excel-only for v1 — a sheet-per-scenario + workbook is an inherently Excel artifact; there is no `format` field (or it is + fixed to `"xlsx"`). CSV degradation is a fast-follow if a warehouse consumer + ever needs it. + +- **Trigger & lifecycle (mirror ADR-0055).** The **front end** creates the + app-owned `tasks` row and writes the **selection recipe** — + `{portfolio_id, scenario_ids, filters, property_ids?, recipient_email}` — into + `tasks.inputs` (JSON, FE-owned), then calls the FastAPI route + (`backend/app/exports/`) with **only the `task_id`**. The route reads the + recipe from `tasks.inputs`, **resolves** it to the concrete internal + `property.id` set (below), **pins the resolved id list + recipe** onto **one + pre-created `sub_task`'s `inputs`**, and drops one SQS message + (`task_id`, `sub_task_id`) on a new `ARA_EXPORT_SQS_URL` queue. The new + `applications/ara_export` Lambda runs in **attach mode**, reading that recipe + from the sub_task; `TaskOrchestrator` owns status + roll-up. Nothing large + ever travels in an HTTP body **or** on SQS — the id list rides + `tasks.inputs` → `sub_task.inputs` (both Postgres JSON), so the 256 KB SQS + limit is irrelevant by construction and the "tens of thousands of ids" + concern is structurally impossible. + +- **Filters are the ADR-0056 shared contract.** The route resolves `filters` + with the **same** `PropertyGroupFilters` model and + `resolve_filtered_property_ids(session, portfolio_id, filters)` function the + Modelling Run Distributor uses — one filter vocabulary, one resolver, two + triggers. New filter dimensions land on `PropertyGroupFilters` and both + triggers inherit them. `property_ids` is an **override, not a co-filter**: + present and non-empty ⇒ export exactly those ids; absent ⇒ resolve from + `filters`. Precedence (not intersection) avoids the "I ticked 5 rows, got 3" + surprise where a lingering filter silently drops hand-picked rows. Both paths + key on internal `property.id` (as `data_exports.py` and the modelling resolver + already do), so identifiers align. + +- **Persisted data only — the Effective EPC, no live API.** Because model A + guarantees every exported Property has a Plan, it has been through Ingestion → + Baseline → Modelling and its cert / Effective EPC / Baseline Performance / + Plans are all persisted. The Export is a pure read. Descriptive fields follow + the **Effective EPC** precedence — `property_overrides` → lodged EPC → + predicted EPC — the same ladder `property_filters._resolved_component_values` + already implements, widened from `property_type`/`built_form` to the full + descriptive set. **No live `EpcClientService` calls**: tens of thousands of + sequential gov-API fetches would blow the Lambda's 15-minute ceiling and + hammer the register. + +- **Promote the data layer into DDD, split by concern.** A `repositories/` + read repository returns the persisted, override-resolved rows for a + `(portfolio_id, scenario_id, property-id set)`; a **pure `domain/` + column-shaper** owns the frozen 21-measure contract and the + measure→cost-column pivot + savings summation (trivially unit-testable). The + `applications/ara_export` Lambda composes: repo read → domain shape → styling + → S3 → email. `scripts/data_exports.py` becomes a thin CLI wrapper over the + same modules (or is retired) so the query/column logic exists **once** — this + also removes the `_PROPERTY_TYPE_CODES` map duplicated today in both + `data_exports.py` and `property_filters.py`. + +- **Frozen column contract, mimic `data_exports.py`.** The 21-measure + `EXPECTED_MEASURE_COLUMNS` contract is **stable** — every measure column + always present, blank where a Property lacks that measure — rather than + emitting only the measures that happen to appear. Property identity + Effective + EPC descriptive fields + pivoted per-measure cost columns + summed savings + + Plan post-works figures (SAP / kWh / CO₂ / bill), per sheet. + +- **Programmatic branding, palette in `infrastructure/`.** Styling is a + **programmatic openpyxl** pass (header fill + bold header font, frozen top + row, banded rows, coloured cost/band columns, column widths), not a populated + template. A2's **dynamic sheet count** defeats a fixed-sheet template (which + would force lossy per-sheet `copy_worksheet` cloning); a helper styles + whatever it generates. The brand palette + styling helper live in + `infrastructure/xlsx/` — filling the "no brand-colour constants exist + anywhere" gap once, reusable enough that `audit_generator` could later adopt + it. `domain/` stays pure and colour-agnostic. Monday.com-style **defaults** + ship now behind a `# TODO(brand)` marker until real Domna hex values land. + +- **Caps & memory.** The bulk-download `MAX_PROPERTIES = 500` (a ZIP-of-GBs + policy) is dropped; the Export payload is one `.xlsx` of mostly numbers. The + route rejects `> ARA_EXPORT_MAX_PROPERTIES` (**100 000**, configurable) + synchronously with a legible "narrow your selection" error. The Lambda builds + in **normal openpyxl mode on a 4 GB-memory Lambda** to start, streamed to a + temp file on `/tmp` and multipart-uploaded (`S3Client.upload_file`), never an + in-memory `BytesIO`. The 100k ceiling in normal mode may need more memory + (openpyxl holds every cell as a Python object — file size is **not** the RAM + gauge); the fallback if it OOMs is to raise the Lambda memory, then switch to + `write_only` mode (which constrains the styling helper to style-at-write). The + common case (hundreds–low-thousands per portfolio) is comfortably within + 4 GB. + +- **Delivery — both channels (ADR-0059).** The workbook is written to the + dedicated `DOCUMENT_EXPORTS_BUCKET` (reused, not the shared `DATA_BUCKET`); + the presigned URL (60-minute expiry, `_URL_TTL_SECONDS = 3600`) and a + summary are written to `sub_task.outputs` **and** emailed via + `SesSmtpEmailSender` to `recipient_email`. Email send is **best-effort** + (wrapped in try/except — a delivery failure does not fail the task; the URL is + still on `sub_task.outputs`, which the FE polls). + +- **Best-effort rows.** A Property/Plan that fails to resolve is skipped and + recorded in `sub_task.outputs`, not a hard failure. The run **fails only** on + an infrastructure error (DB / S3 / workbook / upload) or when the whole + selection yields **zero exportable rows**. + +- **Idempotency.** The `sub_task` is created with the same idempotent + `INSERT ... WHERE NOT EXISTS` the DB arbitrates for double-submit + (`create_download_subtask` pattern). + +## Deferred / future work + +- **Tag-based filtering.** The next filter the front end wants is **tags**, but + there is no tag model in the backend (no `property_tags` table, no + override→EPC→predicted resolution ladder for it) — it is a new concept that + needs its own `/grill-with-docs` before landing on `PropertyGroupFilters`. + Once designed, both the Export and the Modelling Run triggers inherit it via + the ADR-0056 shared contract. A `# TODO(tags)` marker at the filter seam + points back to this ADR. +- **CSV output.** Only if a warehouse/finance consumer needs a flat feed; + would collapse the sheets to one table with a `scenario_id` column, unbranded. +- **`write_only` streaming** if the 100k ceiling proves too heavy for a + reasonably-sized Lambda in normal mode. +- **Exports-bucket retention** — the Exports are transient; an expiry lifecycle + rule is likely warranted but is left to a follow-up (as with ADR-0060). + +## Considered options + +- **Property-scoped export (model B)** — resolve Properties first, attach + scenario columns, keep no-Plan Properties with blank scenario cells. Rejected: + the feature is "give me the modelled results of Scenario X", so a Property with + no Plan for the Scenario has nothing to show; blanks would mislead. +- **One Scenario per export (A1)** — simplest contract, single flat result. + Rejected: the FE screen lets users tick several Scenarios for comparison, which + is exactly the sheet-per-Scenario workbook. +- **Import `scripts/data_exports.py` from the Lambda.** Rejected: re-buries raw + SQL in `scripts/`, keeps the dead live-EPC dependency, duplicates the + code-maps, and can't sit cleanly in the DDD layout. +- **Live EPC fetch (as the script does).** Rejected: tens of thousands of + sequential gov-API calls won't finish in a Lambda and abuse the register; + model A guarantees the data is already persisted. +- **`property_ids` intersects `filters`.** Rejected: a hand-picked selection + silently shrinking because of a stale filter is a surprising, hard-to-debug + UX; override precedence is predictable. +- **Populate a branded `.xlsx` template** (the `audit_generator` pattern). + Rejected for A2: a fixed-sheet template can't hold a dynamic number of + scenario sheets without lossy runtime `copy_worksheet` cloning, and couples the + deploy to a binary asset. +- **`write_only` from day one.** Deferred: it constrains styling to + style-at-write for no benefit at the common scale; adopt only if 4 GB normal + mode proves insufficient at the ceiling. + +## Consequences + +- New DDD pieces: `applications/ara_export/` (thin `@task_handler` + + trigger body carrying only `task_id`/`subtask_id`), + `orchestration/ara_export_orchestrator.py`, a `ScenarioExportRepository` in + `repositories/` returning an override-resolved read-model (so the orchestrator + never names `infrastructure.postgres.*`), a pure column-shaper in `domain/`, + and a brand palette + openpyxl styling helper in `infrastructure/xlsx/`. The + only `backend/` touches are the new `backend/app/exports/` route and reusing + `resolve_filtered_property_ids`. +- New infra: an `ARA_EXPORT_SQS_URL` queue (terraform), the Lambda at 4 GB + memory with `/tmp` for the workbook, reuse of `DOCUMENT_EXPORTS_BUCKET` and the + SES/S3 adapters from ADR-0059. +- The pinned, resolved `property.id` set makes a run reproducible even if the + portfolio changes after submission. +- `PropertyGroupFilters` / `resolve_filtered_property_ids` (ADR-0056) now has a + second production caller; changes to the shared contract affect both triggers — + the intended shareability, made explicit. +- `scripts/data_exports.py`'s query/column logic is de-duplicated into the new + modules; the standalone CLI either wraps them or is retired. From 1a0e87e593d42fbef507a4bd654e856daf1070b8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:08:26 +0000 Subject: [PATCH 02/46] =?UTF-8?q?Pivot=20a=20Property's=20measure=20cost?= =?UTF-8?q?=20onto=20its=20scenario-sheet=20column=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/__init__.py | 0 domain/scenario_export/scenario_sheet.py | 111 ++++++++++++++++++ tests/domain/scenario_export/__init__.py | 0 .../scenario_export/test_scenario_sheet.py | 67 +++++++++++ 4 files changed, 178 insertions(+) create mode 100644 domain/scenario_export/__init__.py create mode 100644 domain/scenario_export/scenario_sheet.py create mode 100644 tests/domain/scenario_export/__init__.py create mode 100644 tests/domain/scenario_export/test_scenario_sheet.py diff --git a/domain/scenario_export/__init__.py b/domain/scenario_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py new file mode 100644 index 000000000..9bdb5ff95 --- /dev/null +++ b/domain/scenario_export/scenario_sheet.py @@ -0,0 +1,111 @@ +"""The Scenario Export sheet shape (ADR-0065). + +Pure logic: given the persisted, override-resolved data for the Properties in one +Scenario, lay out the wide export sheet — property identity + Effective-EPC +descriptive fields, each measure's cost pivoted onto its own column against a +frozen column contract, savings summed per Property, and the Plan's post-works +figures. No I/O: the repository reads the rows and the infrastructure layer +renders the workbook; this module only shapes one sheet's rows. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional, Sequence + +# The measure cost columns always present on every sheet, in a stable order, so +# every scenario sheet in the workbook shares one column contract regardless of +# which measures a given selection happens to contain (ADR-0065). A Property that +# lacks a measure carries a blank in that column, never a missing column. +MEASURE_COLUMNS: tuple[str, ...] = ( + "suspended_floor_insulation", + "solid_floor_insulation", + "external_wall_insulation", + "internal_wall_insulation", + "cavity_wall_insulation", + "loft_insulation", + "flat_roof_insulation", + "room_roof_insulation", + "secondary_glazing", + "double_glazing", + "solar_pv", + "high_heat_retention_storage_heaters", + "air_source_heat_pump", + "boiler_upgrade", + "gas_boiler_upgrade", + "roomstat_programmer_trvs", + "time_temperature_zone_control", + "low_energy_lighting", + "mechanical_ventilation", + "system_tune_up", + "system_tune_up_zoned", +) + + +@dataclass(frozen=True) +class ExportMeasure: + """One selected measure on a Property's default Plan for the Scenario: the + measure type (the pivot column), its installed cost, and the SAP / carbon / + energy / bill savings attributed to it. ``includes_battery`` distinguishes a + solar-with-battery Option so it pivots to its own column (ADR-0065).""" + + measure_type: str + estimated_cost: Optional[float] = None + sap_points: Optional[float] = None + co2_equivalent_savings: Optional[float] = None + kwh_savings: Optional[float] = None + energy_cost_savings: Optional[float] = None + includes_battery: bool = False + + +@dataclass(frozen=True) +class PropertyScenarioData: + """One Property's persisted data for one Scenario: identity, the + Effective-EPC descriptive fields (already override-resolved by the + repository), the default Plan's post-works figures, and the selected + measures. The input row the shaper turns into one wide sheet row.""" + + property_id: int + landlord_property_id: Optional[str] = None + uprn: Optional[str] = None + address: Optional[str] = None + postcode: Optional[str] = None + property_type: Optional[str] = None + walls: Optional[str] = None + roof: Optional[str] = None + floor: Optional[str] = None + windows: Optional[str] = None + heating: Optional[str] = None + hot_water: Optional[str] = None + lighting: Optional[str] = None + total_floor_area: Optional[float] = None + number_of_rooms: Optional[int] = None + lodgement_date: Optional[str] = None + is_expired: Optional[bool] = None + current_epc_rating: Optional[str] = None + current_sap_points: Optional[float] = None + post_sap_points: Optional[float] = None + post_epc_rating: Optional[str] = None + cost_of_works: Optional[float] = None + contingency_cost: Optional[float] = None + co2_savings: Optional[float] = None + energy_bill_savings: Optional[float] = None + energy_consumption_savings: Optional[float] = None + valuation_increase: Optional[float] = None + measures: Sequence[ExportMeasure] = () + + +@dataclass(frozen=True) +class ExportSheet: + """One laid-out scenario sheet: the ordered column contract and the rows, + each a column-keyed mapping ready for the workbook renderer.""" + + columns: tuple[str, ...] + rows: tuple[dict[str, Any], ...] + + +def shape_scenario_sheet( + properties: Sequence[PropertyScenarioData], +) -> ExportSheet: + """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" + raise NotImplementedError diff --git a/tests/domain/scenario_export/__init__.py b/tests/domain/scenario_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py new file mode 100644 index 000000000..01b20f8ed --- /dev/null +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -0,0 +1,67 @@ +"""The Scenario Export sheet shape (ADR-0065) — pure layout of one scenario +sheet: measure costs pivoted onto a frozen column contract, savings summed per +Property, and the Plan's post-works figures.""" + +from typing import Optional, Sequence + +from domain.scenario_export.scenario_sheet import ( + ExportMeasure, + PropertyScenarioData, + shape_scenario_sheet, +) + + +def _measure( + *, + measure_type: str = "loft_insulation", + estimated_cost: Optional[float] = 1000.0, + sap_points: Optional[float] = None, + co2_equivalent_savings: Optional[float] = None, + kwh_savings: Optional[float] = None, + energy_cost_savings: Optional[float] = None, + includes_battery: bool = False, +) -> ExportMeasure: + return ExportMeasure( + measure_type=measure_type, + estimated_cost=estimated_cost, + sap_points=sap_points, + co2_equivalent_savings=co2_equivalent_savings, + kwh_savings=kwh_savings, + energy_cost_savings=energy_cost_savings, + includes_battery=includes_battery, + ) + + +def _property( + *, + property_id: int = 1, + landlord_property_id: Optional[str] = "LP1", + measures: Sequence[ExportMeasure] = (), + **fields: object, +) -> PropertyScenarioData: + return PropertyScenarioData( + property_id=property_id, + landlord_property_id=landlord_property_id, + measures=measures, + **fields, # pyright: ignore[reportArgumentType] + ) + + +def test_a_measures_cost_lands_in_its_own_column_with_the_property_identity() -> None: + # arrange — one Property with one loft-insulation measure priced at £1200. + prop = _property( + property_id=42, + landlord_property_id="LP42", + measures=[_measure(measure_type="loft_insulation", estimated_cost=1200.0)], + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — one row carrying the Property identity and the cost in the + # loft_insulation column. + assert len(sheet.rows) == 1 + row = sheet.rows[0] + assert row["property_id"] == 42 + assert row["landlord_property_id"] == "LP42" + assert row["loft_insulation"] == 1200.0 From c4e1cc4229d25575d295c90edf61ae70c4448355 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:09:00 +0000 Subject: [PATCH 03/46] =?UTF-8?q?Pivot=20a=20Property's=20measure=20cost?= =?UTF-8?q?=20onto=20its=20scenario-sheet=20column=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 9bdb5ff95..7a0ebfb68 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -108,4 +108,14 @@ def shape_scenario_sheet( properties: Sequence[PropertyScenarioData], ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" - raise NotImplementedError + columns: tuple[str, ...] = ("property_id", "landlord_property_id") + rows: list[dict[str, Any]] = [] + for prop in properties: + row: dict[str, Any] = { + "property_id": prop.property_id, + "landlord_property_id": prop.landlord_property_id, + } + for measure in prop.measures: + row[measure.measure_type] = measure.estimated_cost + rows.append(row) + return ExportSheet(columns=columns, rows=tuple(rows)) From 04be6b57a25c095e73d4713817c083072b840243 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:09:40 +0000 Subject: [PATCH 04/46] =?UTF-8?q?Guarantee=20every=20measure=20column=20on?= =?UTF-8?q?=20every=20scenario=20sheet=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index 01b20f8ed..6712074bc 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -5,6 +5,7 @@ Property, and the Plan's post-works figures.""" from typing import Optional, Sequence from domain.scenario_export.scenario_sheet import ( + MEASURE_COLUMNS, ExportMeasure, PropertyScenarioData, shape_scenario_sheet, @@ -65,3 +66,23 @@ def test_a_measures_cost_lands_in_its_own_column_with_the_property_identity() -> assert row["property_id"] == 42 assert row["landlord_property_id"] == "LP42" assert row["loft_insulation"] == 1200.0 + + +def test_every_measure_column_is_present_and_blank_where_the_property_lacks_it() -> None: + # arrange — a Property with a single measure. Every other measure column + # must still appear on the sheet (the frozen contract, ADR-0065), blank for + # this Property, so all scenario sheets share one column set. + prop = _property( + measures=[_measure(measure_type="cavity_wall_insulation", estimated_cost=800.0)] + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — the full frozen measure contract is present; the one selected + # measure carries its cost, every other measure column is blank. + assert set(MEASURE_COLUMNS).issubset(set(sheet.columns)) + row = sheet.rows[0] + assert row["cavity_wall_insulation"] == 800.0 + assert row["loft_insulation"] == "" + assert row["solar_pv"] == "" From 23ff1e73a827416dd46550c9047369437a5db2c1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:10:16 +0000 Subject: [PATCH 05/46] =?UTF-8?q?Guarantee=20every=20measure=20column=20on?= =?UTF-8?q?=20every=20scenario=20sheet=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 7a0ebfb68..3d9dfda7b 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -41,6 +41,9 @@ MEASURE_COLUMNS: tuple[str, ...] = ( "system_tune_up_zoned", ) +# Property identity columns, leading every sheet before the measure columns. +_ID_COLUMNS: tuple[str, ...] = ("property_id", "landlord_property_id") + @dataclass(frozen=True) class ExportMeasure: @@ -108,14 +111,19 @@ def shape_scenario_sheet( properties: Sequence[PropertyScenarioData], ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" - columns: tuple[str, ...] = ("property_id", "landlord_property_id") + columns: tuple[str, ...] = _ID_COLUMNS + MEASURE_COLUMNS rows: list[dict[str, Any]] = [] for prop in properties: row: dict[str, Any] = { "property_id": prop.property_id, "landlord_property_id": prop.landlord_property_id, } + # Every measure column is present, blank unless this Property has it, so + # all scenario sheets share one contract (ADR-0065). + for column in MEASURE_COLUMNS: + row[column] = "" for measure in prop.measures: - row[measure.measure_type] = measure.estimated_cost + if measure.measure_type in MEASURE_COLUMNS: + row[measure.measure_type] = measure.estimated_cost rows.append(row) return ExportSheet(columns=columns, rows=tuple(rows)) From 478ee20aaa7d59d1d0425b35c7e654d7f15a3fff Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:11:01 +0000 Subject: [PATCH 06/46] =?UTF-8?q?Roll=20up=20each=20Property's=20SAP=20poi?= =?UTF-8?q?nts=20and=20savings=20across=20its=20measures=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index 6712074bc..eaa2190ce 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -4,6 +4,8 @@ Property, and the Plan's post-works figures.""" from typing import Optional, Sequence +import pytest + from domain.scenario_export.scenario_sheet import ( MEASURE_COLUMNS, ExportMeasure, @@ -86,3 +88,41 @@ def test_every_measure_column_is_present_and_blank_where_the_property_lacks_it() assert row["cavity_wall_insulation"] == 800.0 assert row["loft_insulation"] == "" assert row["solar_pv"] == "" + + +def test_savings_and_sap_points_are_summed_across_a_propertys_measures() -> None: + # arrange — two measures, each pivoting to its own cost column and each + # contributing SAP points and carbon / energy / bill savings. + prop = _property( + measures=[ + _measure( + measure_type="loft_insulation", + estimated_cost=1000.0, + sap_points=3.0, + co2_equivalent_savings=0.4, + kwh_savings=900.0, + energy_cost_savings=120.0, + ), + _measure( + measure_type="solar_pv", + estimated_cost=5000.0, + sap_points=5.0, + co2_equivalent_savings=1.1, + kwh_savings=2500.0, + energy_cost_savings=300.0, + ), + ] + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — each cost pivots to its own column, and the savings roll up to the + # Property's totals. + row = sheet.rows[0] + assert row["loft_insulation"] == 1000.0 + assert row["solar_pv"] == 5000.0 + assert row["sap_points"] == 8.0 + assert row["co2_equivalent_savings"] == pytest.approx(1.5) + assert row["kwh_savings"] == 3400.0 + assert row["energy_cost_savings"] == 420.0 From b52b38e6f330413ec5c2b66a9cd4cc1938aeb3b5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:11:36 +0000 Subject: [PATCH 07/46] =?UTF-8?q?Roll=20up=20each=20Property's=20SAP=20poi?= =?UTF-8?q?nts=20and=20savings=20across=20its=20measures=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 3d9dfda7b..33871d923 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -44,6 +44,20 @@ MEASURE_COLUMNS: tuple[str, ...] = ( # Property identity columns, leading every sheet before the measure columns. _ID_COLUMNS: tuple[str, ...] = ("property_id", "landlord_property_id") +# Per-Property roll-ups of the selected measures' attributed impact, trailing the +# measure columns (ADR-0065). +_SAVINGS_COLUMNS: tuple[str, ...] = ( + "sap_points", + "co2_equivalent_savings", + "kwh_savings", + "energy_cost_savings", +) + + +def _sum(values: Sequence[Optional[float]]) -> float: + """Sum optional numbers, treating an absent contribution as zero.""" + return sum(value or 0.0 for value in values) + @dataclass(frozen=True) class ExportMeasure: @@ -111,7 +125,7 @@ def shape_scenario_sheet( properties: Sequence[PropertyScenarioData], ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" - columns: tuple[str, ...] = _ID_COLUMNS + MEASURE_COLUMNS + columns: tuple[str, ...] = _ID_COLUMNS + MEASURE_COLUMNS + _SAVINGS_COLUMNS rows: list[dict[str, Any]] = [] for prop in properties: row: dict[str, Any] = { @@ -125,5 +139,14 @@ def shape_scenario_sheet( for measure in prop.measures: if measure.measure_type in MEASURE_COLUMNS: row[measure.measure_type] = measure.estimated_cost + # Roll the selected measures' attributed impact up to the Property. + row["sap_points"] = _sum([m.sap_points for m in prop.measures]) + row["co2_equivalent_savings"] = _sum( + [m.co2_equivalent_savings for m in prop.measures] + ) + row["kwh_savings"] = _sum([m.kwh_savings for m in prop.measures]) + row["energy_cost_savings"] = _sum( + [m.energy_cost_savings for m in prop.measures] + ) rows.append(row) return ExportSheet(columns=columns, rows=tuple(rows)) From df768d882e34cfafe526256bebd4b4e48f7f440b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:11:59 +0000 Subject: [PATCH 08/46] =?UTF-8?q?Total=20each=20Property's=20retrofit=20co?= =?UTF-8?q?st=20from=20its=20measure=20costs=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index eaa2190ce..e21e633a2 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -126,3 +126,19 @@ def test_savings_and_sap_points_are_summed_across_a_propertys_measures() -> None assert row["co2_equivalent_savings"] == pytest.approx(1.5) assert row["kwh_savings"] == 3400.0 assert row["energy_cost_savings"] == 420.0 + + +def test_total_retrofit_cost_sums_the_per_measure_costs() -> None: + # arrange — a Property with two priced measures. + prop = _property( + measures=[ + _measure(measure_type="loft_insulation", estimated_cost=1200.0), + _measure(measure_type="cavity_wall_insulation", estimated_cost=800.0), + ] + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — the package total is the sum of the measure costs. + assert sheet.rows[0]["total_retrofit_cost"] == 2000.0 From a6eaac5fe3b2bc9c4c24006ec5e02e55eaa482ae Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:12:23 +0000 Subject: [PATCH 09/46] =?UTF-8?q?Total=20each=20Property's=20retrofit=20co?= =?UTF-8?q?st=20from=20its=20measure=20costs=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 33871d923..e38116679 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -53,6 +53,9 @@ _SAVINGS_COLUMNS: tuple[str, ...] = ( "energy_cost_savings", ) +# Package-level totals, trailing the savings roll-ups. +_TOTAL_COLUMNS: tuple[str, ...] = ("total_retrofit_cost",) + def _sum(values: Sequence[Optional[float]]) -> float: """Sum optional numbers, treating an absent contribution as zero.""" @@ -125,7 +128,9 @@ def shape_scenario_sheet( properties: Sequence[PropertyScenarioData], ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" - columns: tuple[str, ...] = _ID_COLUMNS + MEASURE_COLUMNS + _SAVINGS_COLUMNS + columns: tuple[str, ...] = ( + _ID_COLUMNS + MEASURE_COLUMNS + _SAVINGS_COLUMNS + _TOTAL_COLUMNS + ) rows: list[dict[str, Any]] = [] for prop in properties: row: dict[str, Any] = { @@ -148,5 +153,6 @@ def shape_scenario_sheet( row["energy_cost_savings"] = _sum( [m.energy_cost_savings for m in prop.measures] ) + row["total_retrofit_cost"] = _sum([m.estimated_cost for m in prop.measures]) rows.append(row) return ExportSheet(columns=columns, rows=tuple(rows)) From f758216bf19c78ad5450294842e78c75e77dd69b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:12:51 +0000 Subject: [PATCH 10/46] =?UTF-8?q?Pivot=20solar=20PV=20with=20a=20battery?= =?UTF-8?q?=20to=20its=20own=20column=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index e21e633a2..54d35e37e 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -142,3 +142,26 @@ def test_total_retrofit_cost_sums_the_per_measure_costs() -> None: # assert — the package total is the sum of the measure costs. assert sheet.rows[0]["total_retrofit_cost"] == 2000.0 + + +def test_solar_pv_with_a_battery_pivots_to_its_own_battery_column() -> None: + # arrange — a solar_pv Option that includes a battery. + prop = _property( + measures=[ + _measure( + measure_type="solar_pv", estimated_cost=6000.0, includes_battery=True + ) + ] + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — the cost lands in the battery variant column (a stable part of the + # contract), the plain solar_pv column stays blank, and the total still + # counts it. + assert "solar_pv_with_battery" in sheet.columns + row = sheet.rows[0] + assert row["solar_pv_with_battery"] == 6000.0 + assert row["solar_pv"] == "" + assert row["total_retrofit_cost"] == 6000.0 From 91c88743f74aa95b0688978d4c5d88f7e32aec71 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:13:22 +0000 Subject: [PATCH 11/46] =?UTF-8?q?Pivot=20solar=20PV=20with=20a=20battery?= =?UTF-8?q?=20to=20its=20own=20column=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index e38116679..64cb5a6d9 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -41,6 +41,14 @@ MEASURE_COLUMNS: tuple[str, ...] = ( "system_tune_up_zoned", ) +# A solar_pv Option that installs a battery pivots to its own column, kept in the +# frozen contract so a sheet with battery-solar shares columns with one without. +_BATTERY_MEASURE_COLUMNS: tuple[str, ...] = ("solar_pv_with_battery",) + +# Every measure cost column present on a sheet: the frozen base contract plus the +# battery variant. +_ALL_MEASURE_COLUMNS: tuple[str, ...] = MEASURE_COLUMNS + _BATTERY_MEASURE_COLUMNS + # Property identity columns, leading every sheet before the measure columns. _ID_COLUMNS: tuple[str, ...] = ("property_id", "landlord_property_id") @@ -62,6 +70,14 @@ def _sum(values: Sequence[Optional[float]]) -> float: return sum(value or 0.0 for value in values) +def _measure_column(measure: ExportMeasure) -> str: + """The pivot column for a measure: a solar_pv Option carrying a battery lands + in its own ``solar_pv_with_battery`` column (ADR-0065).""" + if measure.measure_type == "solar_pv" and measure.includes_battery: + return "solar_pv_with_battery" + return measure.measure_type + + @dataclass(frozen=True) class ExportMeasure: """One selected measure on a Property's default Plan for the Scenario: the @@ -129,7 +145,7 @@ def shape_scenario_sheet( ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" columns: tuple[str, ...] = ( - _ID_COLUMNS + MEASURE_COLUMNS + _SAVINGS_COLUMNS + _TOTAL_COLUMNS + _ID_COLUMNS + _ALL_MEASURE_COLUMNS + _SAVINGS_COLUMNS + _TOTAL_COLUMNS ) rows: list[dict[str, Any]] = [] for prop in properties: @@ -139,11 +155,12 @@ def shape_scenario_sheet( } # Every measure column is present, blank unless this Property has it, so # all scenario sheets share one contract (ADR-0065). - for column in MEASURE_COLUMNS: + for column in _ALL_MEASURE_COLUMNS: row[column] = "" for measure in prop.measures: - if measure.measure_type in MEASURE_COLUMNS: - row[measure.measure_type] = measure.estimated_cost + column = _measure_column(measure) + if column in _ALL_MEASURE_COLUMNS: + row[column] = measure.estimated_cost # Roll the selected measures' attributed impact up to the Property. row["sap_points"] = _sum([m.sap_points for m in prop.measures]) row["co2_equivalent_savings"] = _sum( From 6a6ffb695c1cc1ce2e908750da9e7d2ad8012aef Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:14:13 +0000 Subject: [PATCH 12/46] =?UTF-8?q?Carry=20the=20Plan's=20post-works=20figur?= =?UTF-8?q?es=20onto=20the=20export=20row=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index 54d35e37e..44d756c3d 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -165,3 +165,29 @@ def test_solar_pv_with_a_battery_pivots_to_its_own_battery_column() -> None: assert row["solar_pv_with_battery"] == 6000.0 assert row["solar_pv"] == "" assert row["total_retrofit_cost"] == 6000.0 + + +def test_plan_post_works_figures_come_from_the_plan_row_blank_when_absent() -> None: + # arrange — a Property whose default Plan carries the SAP calculator's + # post-works figures; the contingency figure happens to be absent. + prop = _property( + post_sap_points=78.0, + post_epc_rating="C", + cost_of_works=4200.0, + co2_savings=1.8, + energy_bill_savings=350.0, + energy_consumption_savings=4100.0, + valuation_increase=15000.0, + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — the persisted Plan figures pass through directly (no recompute, + # ADR-0065); an absent figure is blank rather than zero. + row = sheet.rows[0] + assert row["post_sap_points"] == 78.0 + assert row["post_epc_rating"] == "C" + assert row["cost_of_works"] == 4200.0 + assert row["valuation_increase"] == 15000.0 + assert row["contingency_cost"] == "" From c2aca6d2e64bd1068b3cee0f9d23fa221070766d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:14:41 +0000 Subject: [PATCH 13/46] =?UTF-8?q?Carry=20the=20Plan's=20post-works=20figur?= =?UTF-8?q?es=20onto=20the=20export=20row=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 33 +++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 64cb5a6d9..8f7dc681c 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -64,6 +64,24 @@ _SAVINGS_COLUMNS: tuple[str, ...] = ( # Package-level totals, trailing the savings roll-ups. _TOTAL_COLUMNS: tuple[str, ...] = ("total_retrofit_cost",) +# The default Plan's post-works figures, taken straight from the persisted Plan +# (the SAP calculator's output) with no recompute (ADR-0065). +_PLAN_COLUMNS: tuple[str, ...] = ( + "post_sap_points", + "post_epc_rating", + "cost_of_works", + "contingency_cost", + "co2_savings", + "energy_bill_savings", + "energy_consumption_savings", + "valuation_increase", +) + + +def _blank(value: Any) -> Any: + """A missing value renders as a blank cell, not a zero or ``None``.""" + return "" if value is None else value + def _sum(values: Sequence[Optional[float]]) -> float: """Sum optional numbers, treating an absent contribution as zero.""" @@ -145,7 +163,11 @@ def shape_scenario_sheet( ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" columns: tuple[str, ...] = ( - _ID_COLUMNS + _ALL_MEASURE_COLUMNS + _SAVINGS_COLUMNS + _TOTAL_COLUMNS + _ID_COLUMNS + + _ALL_MEASURE_COLUMNS + + _SAVINGS_COLUMNS + + _TOTAL_COLUMNS + + _PLAN_COLUMNS ) rows: list[dict[str, Any]] = [] for prop in properties: @@ -171,5 +193,14 @@ def shape_scenario_sheet( [m.energy_cost_savings for m in prop.measures] ) row["total_retrofit_cost"] = _sum([m.estimated_cost for m in prop.measures]) + # The default Plan's post-works figures, passed through as persisted. + row["post_sap_points"] = _blank(prop.post_sap_points) + row["post_epc_rating"] = _blank(prop.post_epc_rating) + row["cost_of_works"] = _blank(prop.cost_of_works) + row["contingency_cost"] = _blank(prop.contingency_cost) + row["co2_savings"] = _blank(prop.co2_savings) + row["energy_bill_savings"] = _blank(prop.energy_bill_savings) + row["energy_consumption_savings"] = _blank(prop.energy_consumption_savings) + row["valuation_increase"] = _blank(prop.valuation_increase) rows.append(row) return ExportSheet(columns=columns, rows=tuple(rows)) From 6fc94b1cc4951fc0e066464f3f8e89e053ad535a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:15:18 +0000 Subject: [PATCH 14/46] =?UTF-8?q?Carry=20Property=20identity=20and=20Effec?= =?UTF-8?q?tive-EPC=20fields=20onto=20the=20export=20row=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export/test_scenario_sheet.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/domain/scenario_export/test_scenario_sheet.py b/tests/domain/scenario_export/test_scenario_sheet.py index 44d756c3d..bc31d7118 100644 --- a/tests/domain/scenario_export/test_scenario_sheet.py +++ b/tests/domain/scenario_export/test_scenario_sheet.py @@ -191,3 +191,36 @@ def test_plan_post_works_figures_come_from_the_plan_row_blank_when_absent() -> N assert row["cost_of_works"] == 4200.0 assert row["valuation_increase"] == 15000.0 assert row["contingency_cost"] == "" + + +def test_identity_and_effective_epc_descriptive_fields_pass_through() -> None: + # arrange — a Property with identity and (already override-resolved) + # Effective-EPC descriptive fields; the floor description is absent. + prop = _property( + property_id=7, + landlord_property_id="LP7", + uprn="100023", + address="12 Oak Street", + postcode="AB1 2CD", + property_type="House", + walls="Cavity wall, as built, insulated", + roof="Pitched, 250mm loft insulation", + current_epc_rating="D", + current_sap_points=58.0, + ) + + # act + sheet = shape_scenario_sheet([prop]) + + # assert — identity and descriptive fields appear as supplied; an unsupplied + # descriptive field is blank. + row = sheet.rows[0] + assert row["uprn"] == "100023" + assert row["address"] == "12 Oak Street" + assert row["postcode"] == "AB1 2CD" + assert row["property_type"] == "House" + assert row["walls"] == "Cavity wall, as built, insulated" + assert row["roof"] == "Pitched, 250mm loft insulation" + assert row["current_epc_rating"] == "D" + assert row["current_sap_points"] == 58.0 + assert row["floor"] == "" From 9c1e79efc67c2d68aad83c76edc680c2137d2a05 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:15:40 +0000 Subject: [PATCH 15/46] =?UTF-8?q?Carry=20Property=20identity=20and=20Effec?= =?UTF-8?q?tive-EPC=20fields=20onto=20the=20export=20row=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 45 ++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 8f7dc681c..5365f7bca 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -49,8 +49,30 @@ _BATTERY_MEASURE_COLUMNS: tuple[str, ...] = ("solar_pv_with_battery",) # battery variant. _ALL_MEASURE_COLUMNS: tuple[str, ...] = MEASURE_COLUMNS + _BATTERY_MEASURE_COLUMNS -# Property identity columns, leading every sheet before the measure columns. -_ID_COLUMNS: tuple[str, ...] = ("property_id", "landlord_property_id") +# Property identity and Effective-EPC descriptive columns, leading every sheet +# before the measure columns. The descriptive fields are already override-resolved +# by the repository (property_overrides → lodged EPC → predicted, ADR-0065). +_ID_COLUMNS: tuple[str, ...] = ( + "property_id", + "landlord_property_id", + "uprn", + "address", + "postcode", + "property_type", + "walls", + "roof", + "floor", + "windows", + "heating", + "hot_water", + "lighting", + "total_floor_area", + "number_of_rooms", + "lodgement_date", + "is_expired", + "current_epc_rating", + "current_sap_points", +) # Per-Property roll-ups of the selected measures' attributed impact, trailing the # measure columns (ADR-0065). @@ -173,7 +195,24 @@ def shape_scenario_sheet( for prop in properties: row: dict[str, Any] = { "property_id": prop.property_id, - "landlord_property_id": prop.landlord_property_id, + "landlord_property_id": _blank(prop.landlord_property_id), + "uprn": _blank(prop.uprn), + "address": _blank(prop.address), + "postcode": _blank(prop.postcode), + "property_type": _blank(prop.property_type), + "walls": _blank(prop.walls), + "roof": _blank(prop.roof), + "floor": _blank(prop.floor), + "windows": _blank(prop.windows), + "heating": _blank(prop.heating), + "hot_water": _blank(prop.hot_water), + "lighting": _blank(prop.lighting), + "total_floor_area": _blank(prop.total_floor_area), + "number_of_rooms": _blank(prop.number_of_rooms), + "lodgement_date": _blank(prop.lodgement_date), + "is_expired": _blank(prop.is_expired), + "current_epc_rating": _blank(prop.current_epc_rating), + "current_sap_points": _blank(prop.current_sap_points), } # Every measure column is present, blank unless this Property has it, so # all scenario sheets share one contract (ADR-0065). From 65fb4fabadfc4916a1f740a9cd5798abe7b76b11 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:16:27 +0000 Subject: [PATCH 16/46] =?UTF-8?q?Extract=20per-Property=20row=20constructi?= =?UTF-8?q?on=20behind=20a=20shape=20helper=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/scenario_sheet.py | 117 +++++++++++------------ 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 5365f7bca..018b89e11 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -180,66 +180,65 @@ class ExportSheet: rows: tuple[dict[str, Any], ...] +_COLUMNS: tuple[str, ...] = ( + _ID_COLUMNS + _ALL_MEASURE_COLUMNS + _SAVINGS_COLUMNS + _TOTAL_COLUMNS + _PLAN_COLUMNS +) + + +def _shape_row(prop: PropertyScenarioData) -> dict[str, Any]: + """One Property's wide row: identity + Effective-EPC descriptive fields, the + measures pivoted onto their cost columns, the savings rolled up, and the + Plan's post-works figures (ADR-0065).""" + row: dict[str, Any] = { + "property_id": prop.property_id, + "landlord_property_id": _blank(prop.landlord_property_id), + "uprn": _blank(prop.uprn), + "address": _blank(prop.address), + "postcode": _blank(prop.postcode), + "property_type": _blank(prop.property_type), + "walls": _blank(prop.walls), + "roof": _blank(prop.roof), + "floor": _blank(prop.floor), + "windows": _blank(prop.windows), + "heating": _blank(prop.heating), + "hot_water": _blank(prop.hot_water), + "lighting": _blank(prop.lighting), + "total_floor_area": _blank(prop.total_floor_area), + "number_of_rooms": _blank(prop.number_of_rooms), + "lodgement_date": _blank(prop.lodgement_date), + "is_expired": _blank(prop.is_expired), + "current_epc_rating": _blank(prop.current_epc_rating), + "current_sap_points": _blank(prop.current_sap_points), + } + # Every measure column is present, blank unless this Property has it, so all + # scenario sheets share one contract (ADR-0065). + for column in _ALL_MEASURE_COLUMNS: + row[column] = "" + for measure in prop.measures: + column = _measure_column(measure) + if column in _ALL_MEASURE_COLUMNS: + row[column] = measure.estimated_cost + # Roll the selected measures' attributed impact up to the Property. + row["sap_points"] = _sum([m.sap_points for m in prop.measures]) + row["co2_equivalent_savings"] = _sum([m.co2_equivalent_savings for m in prop.measures]) + row["kwh_savings"] = _sum([m.kwh_savings for m in prop.measures]) + row["energy_cost_savings"] = _sum([m.energy_cost_savings for m in prop.measures]) + row["total_retrofit_cost"] = _sum([m.estimated_cost for m in prop.measures]) + # The default Plan's post-works figures, passed through as persisted. + row["post_sap_points"] = _blank(prop.post_sap_points) + row["post_epc_rating"] = _blank(prop.post_epc_rating) + row["cost_of_works"] = _blank(prop.cost_of_works) + row["contingency_cost"] = _blank(prop.contingency_cost) + row["co2_savings"] = _blank(prop.co2_savings) + row["energy_bill_savings"] = _blank(prop.energy_bill_savings) + row["energy_consumption_savings"] = _blank(prop.energy_consumption_savings) + row["valuation_increase"] = _blank(prop.valuation_increase) + return row + + def shape_scenario_sheet( properties: Sequence[PropertyScenarioData], ) -> ExportSheet: """Shape one Scenario's Properties into a wide export sheet (ADR-0065).""" - columns: tuple[str, ...] = ( - _ID_COLUMNS - + _ALL_MEASURE_COLUMNS - + _SAVINGS_COLUMNS - + _TOTAL_COLUMNS - + _PLAN_COLUMNS - ) - rows: list[dict[str, Any]] = [] - for prop in properties: - row: dict[str, Any] = { - "property_id": prop.property_id, - "landlord_property_id": _blank(prop.landlord_property_id), - "uprn": _blank(prop.uprn), - "address": _blank(prop.address), - "postcode": _blank(prop.postcode), - "property_type": _blank(prop.property_type), - "walls": _blank(prop.walls), - "roof": _blank(prop.roof), - "floor": _blank(prop.floor), - "windows": _blank(prop.windows), - "heating": _blank(prop.heating), - "hot_water": _blank(prop.hot_water), - "lighting": _blank(prop.lighting), - "total_floor_area": _blank(prop.total_floor_area), - "number_of_rooms": _blank(prop.number_of_rooms), - "lodgement_date": _blank(prop.lodgement_date), - "is_expired": _blank(prop.is_expired), - "current_epc_rating": _blank(prop.current_epc_rating), - "current_sap_points": _blank(prop.current_sap_points), - } - # Every measure column is present, blank unless this Property has it, so - # all scenario sheets share one contract (ADR-0065). - for column in _ALL_MEASURE_COLUMNS: - row[column] = "" - for measure in prop.measures: - column = _measure_column(measure) - if column in _ALL_MEASURE_COLUMNS: - row[column] = measure.estimated_cost - # Roll the selected measures' attributed impact up to the Property. - row["sap_points"] = _sum([m.sap_points for m in prop.measures]) - row["co2_equivalent_savings"] = _sum( - [m.co2_equivalent_savings for m in prop.measures] - ) - row["kwh_savings"] = _sum([m.kwh_savings for m in prop.measures]) - row["energy_cost_savings"] = _sum( - [m.energy_cost_savings for m in prop.measures] - ) - row["total_retrofit_cost"] = _sum([m.estimated_cost for m in prop.measures]) - # The default Plan's post-works figures, passed through as persisted. - row["post_sap_points"] = _blank(prop.post_sap_points) - row["post_epc_rating"] = _blank(prop.post_epc_rating) - row["cost_of_works"] = _blank(prop.cost_of_works) - row["contingency_cost"] = _blank(prop.contingency_cost) - row["co2_savings"] = _blank(prop.co2_savings) - row["energy_bill_savings"] = _blank(prop.energy_bill_savings) - row["energy_consumption_savings"] = _blank(prop.energy_consumption_savings) - row["valuation_increase"] = _blank(prop.valuation_increase) - rows.append(row) - return ExportSheet(columns=columns, rows=tuple(rows)) + rows = tuple(_shape_row(prop) for prop in properties) + return ExportSheet(columns=_COLUMNS, rows=rows) From d70c1b65cc7c7c2ff4a73b1d907a5de6e7df462a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:26:17 +0000 Subject: [PATCH 17/46] =?UTF-8?q?Read=20a=20scenario's=20default-plan=20pr?= =?UTF-8?q?operties=20as=20export=20rows=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- repositories/scenario_export/__init__.py | 0 .../scenario_export_postgres_repository.py | 28 +++++++++ .../scenario_export_repository.py | 25 ++++++++ .../repositories/scenario_export/__init__.py | 0 ...est_scenario_export_postgres_repository.py | 61 +++++++++++++++++++ 5 files changed, 114 insertions(+) create mode 100644 repositories/scenario_export/__init__.py create mode 100644 repositories/scenario_export/scenario_export_postgres_repository.py create mode 100644 repositories/scenario_export/scenario_export_repository.py create mode 100644 tests/repositories/scenario_export/__init__.py create mode 100644 tests/repositories/scenario_export/test_scenario_export_postgres_repository.py diff --git a/repositories/scenario_export/__init__.py b/repositories/scenario_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py new file mode 100644 index 000000000..ff8326627 --- /dev/null +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Sequence + +from sqlmodel import Session + +from domain.scenario_export.scenario_sheet import PropertyScenarioData +from repositories.scenario_export.scenario_export_repository import ( + ScenarioExportRepository, +) + + +class ScenarioExportPostgresRepository(ScenarioExportRepository): + """Reads Scenario Export rows from Postgres (ADR-0065). Hand-rolled SQL over + the live tables (the ``uploaded_file`` repository precedent), referencing only + the mirrored columns; returns the domain read-model, never ORM rows.""" + + def __init__(self, session: Session) -> None: + self._session = session + + def rows_for( + self, + *, + portfolio_id: int, + scenario_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + raise NotImplementedError diff --git a/repositories/scenario_export/scenario_export_repository.py b/repositories/scenario_export/scenario_export_repository.py new file mode 100644 index 000000000..dfc9cb113 --- /dev/null +++ b/repositories/scenario_export/scenario_export_repository.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Sequence + +from domain.scenario_export.scenario_sheet import PropertyScenarioData + + +class ScenarioExportRepository(ABC): + """Reads the persisted data one Scenario Export sheet needs (ADR-0065).""" + + @abstractmethod + def rows_for( + self, + *, + portfolio_id: int, + scenario_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + """The export rows for the Properties in ``property_ids`` that have a + default Plan under the Scenario — identity, override-resolved + Effective-EPC descriptive fields, the Plan's post-works figures, and the + selected measures. A Property with no default Plan for the Scenario is + absent (scenario-scoped, model A).""" + ... diff --git a/tests/repositories/scenario_export/__init__.py b/tests/repositories/scenario_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py new file mode 100644 index 000000000..993e3dd42 --- /dev/null +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -0,0 +1,61 @@ +"""Behaviour of the Postgres-backed ScenarioExportRepository (ADR-0065): reading +one Scenario's Properties — those with a default Plan under it — as the export +read-model, from persisted data only (no live EPC calls).""" + +from __future__ import annotations + +from sqlalchemy import Engine +from sqlmodel import Session + +from datatypes.epc.domain.epc import Epc +from infrastructure.postgres.modelling import PlanModel # registers `plan` +from infrastructure.postgres.property_table import PropertyRow # registers `property` +from repositories.scenario_export.scenario_export_postgres_repository import ( + ScenarioExportPostgresRepository, +) + + +def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( + db_engine: Engine, +) -> None: + # arrange — a Property with a default Plan under scenario 5 carrying the SAP + # calculator's post-works figures. + with Session(db_engine) as session: + session.add( + PropertyRow( + id=1, + portfolio_id=100, + landlord_property_id="LP1", + uprn=100023, + address="12 Oak Street", + postcode="AB1 2CD", + ) + ) + session.add( + PlanModel( + portfolio_id=100, + property_id=1, + scenario_id=5, + is_default=True, + post_sap_points=78.0, + post_epc_rating=Epc.C, + cost_of_works=4200.0, + ) + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — one read-model row carrying identity and the Plan's post-works + # figures. + assert len(rows) == 1 + row = rows[0] + assert row.property_id == 1 + assert row.landlord_property_id == "LP1" + assert row.uprn == "100023" + assert row.post_sap_points == 78.0 + assert row.cost_of_works == 4200.0 From 1bb1e605e2f72eada8c1dd8428e93e7f1fc0dd49 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:26:51 +0000 Subject: [PATCH 18/46] =?UTF-8?q?Read=20a=20scenario's=20default-plan=20pr?= =?UTF-8?q?operties=20as=20export=20rows=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export_postgres_repository.py | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index ff8326627..1b6ca3315 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Sequence +from typing import Any, Optional, Sequence +from sqlalchemy import text from sqlmodel import Session from domain.scenario_export.scenario_sheet import PropertyScenarioData @@ -9,6 +10,35 @@ from repositories.scenario_export.scenario_export_repository import ( ScenarioExportRepository, ) +# The default Plan per (scenario, property), scoped to the requested Properties. +# The JOIN to `plan` is the model-A scenario-scoping (ADR-0065): a Property with +# no default Plan under the Scenario yields no row and drops out. +_ROWS_SQL = text( + "SELECT p.id, p.landlord_property_id, p.uprn, p.address, p.postcode," + " pl.post_sap_points, pl.post_epc_rating, pl.cost_of_works," + " pl.contingency_cost, pl.co2_savings, pl.energy_bill_savings," + " pl.energy_consumption_savings, pl.valuation_increase" + " FROM property p" + " JOIN plan pl ON pl.property_id = p.id" + " WHERE p.portfolio_id = :portfolio_id" + " AND pl.scenario_id = :scenario_id" + " AND pl.is_default = TRUE" + " AND p.id = ANY(:property_ids)" + " ORDER BY p.id" +) + + +def _str(value: Any) -> Optional[str]: + """Render a persisted scalar (int UPRN, Epc enum band) as its string form, + preserving absence as ``None``.""" + if value is None: + return None + return str(getattr(value, "value", value)) + + +def _float(value: Any) -> Optional[float]: + return None if value is None else float(value) + class ScenarioExportPostgresRepository(ScenarioExportRepository): """Reads Scenario Export rows from Postgres (ADR-0065). Hand-rolled SQL over @@ -25,4 +55,31 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): scenario_id: int, property_ids: Sequence[int], ) -> list[PropertyScenarioData]: - raise NotImplementedError + if not property_ids: + return [] + result = self._session.connection().execute( + _ROWS_SQL, + { + "portfolio_id": portfolio_id, + "scenario_id": scenario_id, + "property_ids": list(property_ids), + }, + ) + return [ + PropertyScenarioData( + property_id=row[0], + landlord_property_id=_str(row[1]), + uprn=_str(row[2]), + address=_str(row[3]), + postcode=_str(row[4]), + post_sap_points=_float(row[5]), + post_epc_rating=_str(row[6]), + cost_of_works=_float(row[7]), + contingency_cost=_float(row[8]), + co2_savings=_float(row[9]), + energy_bill_savings=_float(row[10]), + energy_consumption_savings=_float(row[11]), + valuation_increase=_float(row[12]), + ) + for row in result.all() + ] From 14763638e20d74f48e49239ef687aa3e8e409b67 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:27:39 +0000 Subject: [PATCH 19/46] =?UTF-8?q?Lock=20model-A=20scoping:=20exclude=20pro?= =?UTF-8?q?perties=20without=20a=20default=20plan=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- ...est_scenario_export_postgres_repository.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index 993e3dd42..68e3dfa8f 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -59,3 +59,35 @@ def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( assert row.uprn == "100023" assert row.post_sap_points == 78.0 assert row.cost_of_works == 4200.0 + + +def test_excludes_requested_properties_without_a_default_plan_under_the_scenario( + db_engine: Engine, +) -> None: + # arrange — four requested Properties: only #1 has a default Plan under + # scenario 5. #2 has a non-default Plan there, #3 a default Plan under a + # different scenario, #4 no Plan at all. + with Session(db_engine) as session: + for pid in (1, 2, 3, 4): + session.add( + PropertyRow(id=pid, portfolio_id=100, landlord_property_id=f"LP{pid}") + ) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + session.add( + PlanModel(portfolio_id=100, property_id=2, scenario_id=5, is_default=False) + ) + session.add( + PlanModel(portfolio_id=100, property_id=3, scenario_id=9, is_default=True) + ) + session.commit() + + # act — request all four. + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1, 2, 3, 4] + ) + + # assert — only #1 (a default Plan under scenario 5) survives (model A). + assert [r.property_id for r in rows] == [1] From 5dab7beae549cd50d9270ec535ca51f3d849580f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:29:26 +0000 Subject: [PATCH 20/46] =?UTF-8?q?Map=20the=20default=20plan's=20selected?= =?UTF-8?q?=20measures=20onto=20the=20export=20row=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/postgres/product_table.py | 3 + ...est_scenario_export_postgres_repository.py | 94 ++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/infrastructure/postgres/product_table.py b/infrastructure/postgres/product_table.py index b353b3006..fe2639d17 100644 --- a/infrastructure/postgres/product_table.py +++ b/infrastructure/postgres/product_table.py @@ -22,3 +22,6 @@ class MaterialRow(SQLModel, table=True): cost_unit: Optional[str] = Field(default=None) description: Optional[str] = Field(default=None) is_active: bool = Field(default=True) + # Whether a solar_pv Option's product bundle includes a battery; the Scenario + # Export reads it to pivot solar-with-battery to its own column (ADR-0065). + includes_battery: bool = Field(default=False) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index 68e3dfa8f..a6199f32f 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -8,7 +8,11 @@ from sqlalchemy import Engine from sqlmodel import Session from datatypes.epc.domain.epc import Epc -from infrastructure.postgres.modelling import PlanModel # registers `plan` +from infrastructure.postgres.modelling import ( # registers `plan` + `recommendation` + PlanModel, + RecommendationModel, +) +from infrastructure.postgres.product_table import MaterialRow # registers `material` from infrastructure.postgres.property_table import PropertyRow # registers `property` from repositories.scenario_export.scenario_export_postgres_repository import ( ScenarioExportPostgresRepository, @@ -91,3 +95,91 @@ def test_excludes_requested_properties_without_a_default_plan_under_the_scenario # assert — only #1 (a default Plan under scenario 5) survives (model A). assert [r.property_id for r in rows] == [1] + + +def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( + db_engine: Engine, +) -> None: + # arrange — a default Plan with two selected measures (one solar_pv whose + # material carries a battery), plus a non-default and an already-installed + # recommendation that must be excluded. + with Session(db_engine) as session: + session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PlanModel( + id=10, portfolio_id=100, property_id=1, scenario_id=5, is_default=True + ) + ) + session.add(MaterialRow(id=1, type="loft_insulation", includes_battery=False)) + session.add(MaterialRow(id=2, type="solar_pv", includes_battery=True)) + session.add( + RecommendationModel( + plan_id=10, + property_id=1, + type="loft_insulation", + measure_type="loft_insulation", + description="Loft insulation", + estimated_cost=1200.0, + sap_points=3.0, + co2_equivalent_savings=0.4, + kwh_savings=900.0, + energy_cost_savings=120.0, + material_id=1, + default=True, + already_installed=False, + ) + ) + session.add( + RecommendationModel( + plan_id=10, + property_id=1, + type="solar_pv", + measure_type="solar_pv", + description="Solar PV with battery", + estimated_cost=6000.0, + sap_points=5.0, + material_id=2, + default=True, + already_installed=False, + ) + ) + session.add( + RecommendationModel( + plan_id=10, + property_id=1, + type="double_glazing", + measure_type="double_glazing", + description="Double glazing (not selected)", + estimated_cost=4000.0, + default=False, + already_installed=False, + ) + ) + session.add( + RecommendationModel( + plan_id=10, + property_id=1, + type="cavity_wall_insulation", + measure_type="cavity_wall_insulation", + description="CWI (already installed)", + estimated_cost=800.0, + default=True, + already_installed=True, + ) + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — only the two selected measures map through, the battery flag is + # resolved from the material, and savings are carried per measure. + by_type = {m.measure_type: m for m in rows[0].measures} + assert set(by_type) == {"loft_insulation", "solar_pv"} + assert by_type["loft_insulation"].estimated_cost == 1200.0 + assert by_type["loft_insulation"].kwh_savings == 900.0 + assert by_type["loft_insulation"].includes_battery is False + assert by_type["solar_pv"].includes_battery is True From 432898444c746648f45a8bbe163ed47bc2f6651a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:30:11 +0000 Subject: [PATCH 21/46] =?UTF-8?q?Map=20the=20default=20plan's=20selected?= =?UTF-8?q?=20measures=20onto=20the=20export=20row=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export_postgres_repository.py | 56 ++++++++++++++++--- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 1b6ca3315..dabbb52f6 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -5,7 +5,7 @@ from typing import Any, Optional, Sequence from sqlalchemy import text from sqlmodel import Session -from domain.scenario_export.scenario_sheet import PropertyScenarioData +from domain.scenario_export.scenario_sheet import ExportMeasure, PropertyScenarioData from repositories.scenario_export.scenario_export_repository import ( ScenarioExportRepository, ) @@ -27,6 +27,25 @@ _ROWS_SQL = text( " ORDER BY p.id" ) +# The selected measures on each Property's default Plan: the recommendations +# kept in the Optimised Package (``default`` and not ``already_installed``), +# with the installed material's battery flag for the solar pivot (ADR-0065). +# ``default`` is a reserved word, hence the quoting. +_MEASURES_SQL = text( + "SELECT r.property_id, r.measure_type, r.estimated_cost, r.sap_points," + " r.co2_equivalent_savings, r.kwh_savings, r.energy_cost_savings," + " COALESCE(m.includes_battery, FALSE) AS includes_battery" + " FROM recommendation r" + " JOIN plan pl ON pl.id = r.plan_id" + " LEFT JOIN material m ON m.id = r.material_id" + " WHERE pl.portfolio_id = :portfolio_id" + " AND pl.scenario_id = :scenario_id" + " AND pl.is_default = TRUE" + " AND pl.property_id = ANY(:property_ids)" + ' AND r."default" = TRUE' + " AND r.already_installed = FALSE" +) + def _str(value: Any) -> Optional[str]: """Render a persisted scalar (int UPRN, Epc enum band) as its string form, @@ -57,14 +76,13 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): ) -> list[PropertyScenarioData]: if not property_ids: return [] - result = self._session.connection().execute( - _ROWS_SQL, - { - "portfolio_id": portfolio_id, - "scenario_id": scenario_id, - "property_ids": list(property_ids), - }, - ) + params: dict[str, Any] = { + "portfolio_id": portfolio_id, + "scenario_id": scenario_id, + "property_ids": list(property_ids), + } + measures = self._measures_by_property(params) + result = self._session.connection().execute(_ROWS_SQL, params) return [ PropertyScenarioData( property_id=row[0], @@ -80,6 +98,26 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): energy_bill_savings=_float(row[10]), energy_consumption_savings=_float(row[11]), valuation_increase=_float(row[12]), + measures=measures.get(row[0], ()), ) for row in result.all() ] + + def _measures_by_property( + self, params: dict[str, Any] + ) -> dict[int, list[ExportMeasure]]: + result = self._session.connection().execute(_MEASURES_SQL, params) + by_property: dict[int, list[ExportMeasure]] = {} + for row in result.all(): + by_property.setdefault(row[0], []).append( + ExportMeasure( + measure_type=_str(row[1]) or "", + estimated_cost=_float(row[2]), + sap_points=_float(row[3]), + co2_equivalent_savings=_float(row[4]), + kwh_savings=_float(row[5]), + energy_cost_savings=_float(row[6]), + includes_battery=bool(row[7]), + ) + ) + return by_property From ed690d9e36b58f0f8b10220a8e122394ec249eb3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:36:43 +0000 Subject: [PATCH 22/46] =?UTF-8?q?Populate=20export=20rows=20with=20the=20l?= =?UTF-8?q?odged=20EPC's=20descriptive=20fields=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- ...est_scenario_export_postgres_repository.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index a6199f32f..bcd0e8037 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -4,21 +4,103 @@ read-model, from persisted data only (no live EPC calls).""" from __future__ import annotations +from typing import Optional + from sqlalchemy import Engine from sqlmodel import Session from datatypes.epc.domain.epc import Epc +from infrastructure.postgres.epc_property_table import ( # registers epc tables + EpcEnergyElementModel, + EpcPropertyEnergyPerformanceModel, + EpcPropertyModel, +) from infrastructure.postgres.modelling import ( # registers `plan` + `recommendation` PlanModel, RecommendationModel, ) from infrastructure.postgres.product_table import MaterialRow # registers `material` +from infrastructure.postgres.property_override_table import ( # registers overrides + PropertyOverrideRow, +) from infrastructure.postgres.property_table import PropertyRow # registers `property` from repositories.scenario_export.scenario_export_postgres_repository import ( ScenarioExportPostgresRepository, ) +def _add_epc( + session: Session, + *, + property_id: int, + epc_id: int, + source: str = "lodged", + property_type: Optional[str] = None, + total_floor_area_m2: float = 90.0, + registration_date: str = "2005-01-01", + habitable_rooms_count: int = 4, + energy_rating_current: Optional[int] = 58, + band: Optional[str] = "D", + elements: Optional[dict[str, str]] = None, +) -> None: + """Seed one EPC slot (lodged/predicted) with its energy-performance child and + descriptive energy elements. Fills every NOT-NULL column with a benign + default so the test only has to state the fields it cares about.""" + session.add( + EpcPropertyModel( + id=epc_id, + property_id=property_id, + source=source, + property_type=property_type, + total_floor_area_m2=total_floor_area_m2, + registration_date=registration_date, + habitable_rooms_count=habitable_rooms_count, + dwelling_type="House", + tenure="owner-occupied", + transaction_type="none", + inspection_date=registration_date, + solar_water_heating=False, + has_hot_water_cylinder=False, + has_fixed_air_conditioning=False, + door_count=1, + wet_rooms_count=1, + extensions_count=0, + heated_rooms_count=1, + open_chimneys_count=0, + insulated_door_count=0, + cfl_fixed_lighting_bulbs_count=0, + led_fixed_lighting_bulbs_count=0, + incandescent_fixed_lighting_bulbs_count=0, + energy_gas_connection_available=False, + energy_meter_type="standard", + energy_pv_battery_count=0, + energy_wind_turbines_count=0, + energy_gas_smart_meter_present=False, + energy_is_dwelling_export_capable=False, + energy_wind_turbines_terrain_type="none", + energy_electricity_smart_meter_present=False, + ) + ) + session.flush() # parent epc_property before its FK children + session.add( + EpcPropertyEnergyPerformanceModel( + epc_property_id=epc_id, + energy_rating_current=energy_rating_current, + current_energy_efficiency_band=band, + ) + ) + for element_type, description in (elements or {}).items(): + session.add( + EpcEnergyElementModel( + epc_property_id=epc_id, + element_type=element_type, + description=description, + energy_efficiency_rating=3, + environmental_efficiency_rating=3, + ) + ) + + def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( db_engine: Engine, ) -> None: @@ -183,3 +265,55 @@ def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( assert by_type["loft_insulation"].kwh_savings == 900.0 assert by_type["loft_insulation"].includes_battery is False assert by_type["solar_pv"].includes_battery is True + + +def test_effective_epc_descriptive_fields_come_from_the_lodged_epc( + db_engine: Engine, +) -> None: + # arrange — a Property with a default Plan under scenario 5 and a lodged EPC + # carrying descriptive energy elements and current performance. + with Session(db_engine) as session: + session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc( + session, + property_id=1, + epc_id=1, + source="lodged", + property_type="House", + total_floor_area_m2=92.5, + registration_date="2005-01-01", + habitable_rooms_count=5, + energy_rating_current=58, + band="D", + elements={ + "wall": "Cavity wall, as built, insulated", + "roof": "Pitched, 250mm loft insulation", + "main_heating": "Boiler and radiators, mains gas", + "window": "Fully double glazed", + }, + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — descriptive prose, current performance, floor area, lodgement and + # property_type all come from the persisted lodged EPC (no live calls). + row = rows[0] + assert row.walls == "Cavity wall, as built, insulated" + assert row.roof == "Pitched, 250mm loft insulation" + assert row.heating == "Boiler and radiators, mains gas" + assert row.windows == "Fully double glazed" + assert row.property_type == "House" + assert row.total_floor_area == 92.5 + assert row.number_of_rooms == 5 + assert row.current_epc_rating == "D" + assert row.current_sap_points == 58 + assert row.lodgement_date == "2005-01-01" + assert row.is_expired is True From 6ea3f31d822b0ceb12b2246ea1c77d51390d3996 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:38:22 +0000 Subject: [PATCH 23/46] =?UTF-8?q?Populate=20export=20rows=20with=20the=20l?= =?UTF-8?q?odged=20EPC's=20descriptive=20fields=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export_postgres_repository.py | 145 +++++++++++++++--- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index dabbb52f6..208a7bddc 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import date, datetime from typing import Any, Optional, Sequence from sqlalchemy import text @@ -46,6 +47,53 @@ _MEASURES_SQL = text( " AND r.already_installed = FALSE" ) +# The Effective-EPC descriptive picture, read from the persisted lodged EPC slot +# (ADR-0065). Header facts, descriptive energy elements, and current performance +# are read separately and composed per Property. +_EPC_HEADER_SQL = text( + "SELECT property_id, property_type, total_floor_area_m2, registration_date," + " habitable_rooms_count" + " FROM epc_property" + " WHERE property_id = ANY(:property_ids) AND source = 'lodged'" +) +_EPC_ELEMENTS_SQL = text( + "SELECT ep.property_id, e.element_type, e.description" + " FROM epc_energy_element e" + " JOIN epc_property ep ON ep.id = e.epc_property_id" + " WHERE ep.property_id = ANY(:property_ids) AND ep.source = 'lodged'" +) +_EPC_PERF_SQL = text( + "SELECT ep.property_id, perf.energy_rating_current," + " perf.current_energy_efficiency_band" + " FROM epc_property_energy_performance perf" + " JOIN epc_property ep ON ep.id = perf.epc_property_id" + " WHERE ep.property_id = ANY(:property_ids) AND ep.source = 'lodged'" +) + +# EPC energy-element type → export descriptive column. +_ELEMENT_TO_FIELD: dict[str, str] = { + "wall": "walls", + "roof": "roof", + "floor": "floor", + "window": "windows", + "main_heating": "heating", + "hot_water": "hot_water", + "lighting": "lighting", +} + +# A cert older than this many years is flagged expired (as data_exports did). +_EXPIRY_DAYS = 365 * 10 + + +def _is_expired(registration_date: Optional[str]) -> Optional[bool]: + if not registration_date: + return None + try: + lodged = datetime.fromisoformat(registration_date[:10]).date() + except ValueError: + return None + return (date.today() - lodged).days > _EXPIRY_DAYS + def _str(value: Any) -> Optional[str]: """Render a persisted scalar (int UPRN, Epc enum band) as its string form, @@ -82,26 +130,44 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): "property_ids": list(property_ids), } measures = self._measures_by_property(params) + epc = self._effective_epc_by_property(params) result = self._session.connection().execute(_ROWS_SQL, params) - return [ - PropertyScenarioData( - property_id=row[0], - landlord_property_id=_str(row[1]), - uprn=_str(row[2]), - address=_str(row[3]), - postcode=_str(row[4]), - post_sap_points=_float(row[5]), - post_epc_rating=_str(row[6]), - cost_of_works=_float(row[7]), - contingency_cost=_float(row[8]), - co2_savings=_float(row[9]), - energy_bill_savings=_float(row[10]), - energy_consumption_savings=_float(row[11]), - valuation_increase=_float(row[12]), - measures=measures.get(row[0], ()), + rows: list[PropertyScenarioData] = [] + for row in result.all(): + facts = epc.get(row[0], {}) + rows.append( + PropertyScenarioData( + property_id=row[0], + landlord_property_id=_str(row[1]), + uprn=_str(row[2]), + address=_str(row[3]), + postcode=_str(row[4]), + post_sap_points=_float(row[5]), + post_epc_rating=_str(row[6]), + cost_of_works=_float(row[7]), + contingency_cost=_float(row[8]), + co2_savings=_float(row[9]), + energy_bill_savings=_float(row[10]), + energy_consumption_savings=_float(row[11]), + valuation_increase=_float(row[12]), + property_type=facts.get("property_type"), + walls=facts.get("walls"), + roof=facts.get("roof"), + floor=facts.get("floor"), + windows=facts.get("windows"), + heating=facts.get("heating"), + hot_water=facts.get("hot_water"), + lighting=facts.get("lighting"), + total_floor_area=facts.get("total_floor_area"), + number_of_rooms=facts.get("number_of_rooms"), + lodgement_date=facts.get("lodgement_date"), + is_expired=facts.get("is_expired"), + current_epc_rating=facts.get("current_epc_rating"), + current_sap_points=facts.get("current_sap_points"), + measures=measures.get(row[0], ()), + ) ) - for row in result.all() - ] + return rows def _measures_by_property( self, params: dict[str, Any] @@ -121,3 +187,46 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): ) ) return by_property + + def _effective_epc_by_property( + self, params: dict[str, Any] + ) -> dict[int, dict[str, Any]]: + connection = self._session.connection() + facts: dict[int, dict[str, Any]] = {} + + for row in connection.execute(_EPC_HEADER_SQL, params).all(): + registration_date = _str(row[3]) + facts.setdefault(row[0], {}).update( + { + "property_type": _str(row[1]), + "total_floor_area": _float(row[2]), + "lodgement_date": registration_date, + "is_expired": _is_expired(registration_date), + "number_of_rooms": row[4], + } + ) + + # A Property can lodge several rows of one element type (e.g. two walls); + # join their descriptions, as data_exports did. + descriptions: dict[tuple[int, str], list[str]] = {} + for row in connection.execute(_EPC_ELEMENTS_SQL, params).all(): + element_type = _str(row[1]) or "" + descriptions.setdefault((row[0], element_type), []).append( + _str(row[2]) or "" + ) + for (property_id, element_type), descs in descriptions.items(): + field = _ELEMENT_TO_FIELD.get(element_type) + if field is not None: + facts.setdefault(property_id, {})[field] = "; ".join( + d for d in descs if d + ) + + for row in connection.execute(_EPC_PERF_SQL, params).all(): + facts.setdefault(row[0], {}).update( + { + "current_sap_points": row[1], + "current_epc_rating": _str(row[2]), + } + ) + + return facts From 642789db015971399b9d529d8dcb1db3eef9e506 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:39:45 +0000 Subject: [PATCH 24/46] =?UTF-8?q?Resolve=20Effective-EPC=20fields=20by=20o?= =?UTF-8?q?verride=20then=20lodged=20then=20predicted=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- ...est_scenario_export_postgres_repository.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index bcd0e8037..b5940eb33 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -317,3 +317,66 @@ def test_effective_epc_descriptive_fields_come_from_the_lodged_epc( assert row.current_sap_points == 58 assert row.lodgement_date == "2005-01-01" assert row.is_expired is True + + +def test_property_type_override_beats_the_lodged_epc(db_engine: Engine) -> None: + # arrange — the lodged EPC says Flat, but a landlord override says House + # (Effective-EPC precedence: override → lodged, ADR-0065). + with Session(db_engine) as session: + session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc(session, property_id=1, epc_id=1, source="lodged", property_type="Flat") + session.add( + PropertyOverrideRow( + property_id=1, + portfolio_id=100, + building_part=0, + override_component="property_type", + override_value="House", + original_spreadsheet_description="detached house", + ) + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the override wins. + assert rows[0].property_type == "House" + + +def test_falls_back_to_the_predicted_epc_when_there_is_no_lodged_epc( + db_engine: Engine, +) -> None: + # arrange — the Property has only a predicted EPC slot, no lodged cert. + with Session(db_engine) as session: + session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc( + session, + property_id=1, + epc_id=1, + source="predicted", + property_type="Bungalow", + elements={"wall": "Predicted solid brick wall"}, + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the predicted slot supplies the descriptive picture (precedence + # falls through to predicted when no lodged EPC exists). + row = rows[0] + assert row.property_type == "Bungalow" + assert row.walls == "Predicted solid brick wall" From 183d14f5f4a77076f4ea83e166381be8757d8f72 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Jul 2026 23:40:53 +0000 Subject: [PATCH 25/46] =?UTF-8?q?Resolve=20Effective-EPC=20fields=20by=20o?= =?UTF-8?q?verride=20then=20lodged=20then=20predicted=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scenario_export_postgres_repository.py | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 208a7bddc..4029b41eb 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -54,20 +54,29 @@ _EPC_HEADER_SQL = text( "SELECT property_id, property_type, total_floor_area_m2, registration_date," " habitable_rooms_count" " FROM epc_property" - " WHERE property_id = ANY(:property_ids) AND source = 'lodged'" + " WHERE property_id = ANY(:property_ids) AND source = :source" ) _EPC_ELEMENTS_SQL = text( "SELECT ep.property_id, e.element_type, e.description" " FROM epc_energy_element e" " JOIN epc_property ep ON ep.id = e.epc_property_id" - " WHERE ep.property_id = ANY(:property_ids) AND ep.source = 'lodged'" + " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" ) _EPC_PERF_SQL = text( "SELECT ep.property_id, perf.energy_rating_current," " perf.current_energy_efficiency_band" " FROM epc_property_energy_performance perf" " JOIN epc_property ep ON ep.id = perf.epc_property_id" - " WHERE ep.property_id = ANY(:property_ids) AND ep.source = 'lodged'" + " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" +) + +# A landlord property_type override wins over any EPC-derived value (ADR-0065); +# the descriptive prose fields have no override prose, so only property_type is +# resolved this way (the override enum carries coded fabric facts, not prose). +_OVERRIDE_PROPERTY_TYPE_SQL = text( + "SELECT property_id, override_value FROM property_overrides" + " WHERE property_id = ANY(:property_ids) AND building_part = 0" + " AND override_component = 'property_type'" ) # EPC energy-element type → export descriptive column. @@ -190,11 +199,37 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): def _effective_epc_by_property( self, params: dict[str, Any] + ) -> dict[int, dict[str, Any]]: + """The override-resolved Effective-EPC descriptive picture per Property: + override → lodged → predicted (ADR-0065).""" + lodged = self._epc_facts_for_source(params, "lodged") + predicted = self._epc_facts_for_source(params, "predicted") + + facts: dict[int, dict[str, Any]] = {} + for property_id in set(lodged) | set(predicted): + merged = dict(predicted.get(property_id, {})) + # A lodged cert, where present, wins field by field (a null lodged + # field falls through to the predicted value). + for key, value in lodged.get(property_id, {}).items(): + if value is not None: + merged[key] = value + facts[property_id] = merged + + for row in self._session.connection().execute( + _OVERRIDE_PROPERTY_TYPE_SQL, params + ).all(): + facts.setdefault(row[0], {})["property_type"] = _str(row[1]) + + return facts + + def _epc_facts_for_source( + self, params: dict[str, Any], source: str ) -> dict[int, dict[str, Any]]: connection = self._session.connection() + source_params: dict[str, Any] = {**params, "source": source} facts: dict[int, dict[str, Any]] = {} - for row in connection.execute(_EPC_HEADER_SQL, params).all(): + for row in connection.execute(_EPC_HEADER_SQL, source_params).all(): registration_date = _str(row[3]) facts.setdefault(row[0], {}).update( { @@ -209,7 +244,7 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): # A Property can lodge several rows of one element type (e.g. two walls); # join their descriptions, as data_exports did. descriptions: dict[tuple[int, str], list[str]] = {} - for row in connection.execute(_EPC_ELEMENTS_SQL, params).all(): + for row in connection.execute(_EPC_ELEMENTS_SQL, source_params).all(): element_type = _str(row[1]) or "" descriptions.setdefault((row[0], element_type), []).append( _str(row[2]) or "" @@ -221,7 +256,7 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): d for d in descs if d ) - for row in connection.execute(_EPC_PERF_SQL, params).all(): + for row in connection.execute(_EPC_PERF_SQL, source_params).all(): facts.setdefault(row[0], {}).update( { "current_sap_points": row[1], From 62451aa3c9b888ef9c536d20d08c60fad1c9ef2e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:18:59 +0000 Subject: [PATCH 26/46] =?UTF-8?q?Render=20the=20scenario=20export=20workbo?= =?UTF-8?q?ok=20with=20a=20sheet=20per=20scenario=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/xlsx/__init__.py | 0 .../xlsx/scenario_export_workbook.py | 20 +++++++++++++ tests/infrastructure/xlsx/__init__.py | 0 .../xlsx/test_scenario_export_workbook.py | 30 +++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 infrastructure/xlsx/__init__.py create mode 100644 infrastructure/xlsx/scenario_export_workbook.py create mode 100644 tests/infrastructure/xlsx/__init__.py create mode 100644 tests/infrastructure/xlsx/test_scenario_export_workbook.py diff --git a/infrastructure/xlsx/__init__.py b/infrastructure/xlsx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infrastructure/xlsx/scenario_export_workbook.py b/infrastructure/xlsx/scenario_export_workbook.py new file mode 100644 index 000000000..0da8a6f4f --- /dev/null +++ b/infrastructure/xlsx/scenario_export_workbook.py @@ -0,0 +1,20 @@ +"""Renders the Scenario Export workbook (ADR-0065). + +One ``.xlsx`` with one sheet per Scenario (model A2), each sheet's rows shaped by +``domain.scenario_export`` and painted with the Monday-style branding in +``infrastructure.xlsx.branding``. Programmatic openpyxl (no template) so a +dynamic number of scenario sheets is handled natively. +""" + +from __future__ import annotations + +from typing import Sequence + +from domain.scenario_export.scenario_sheet import ExportSheet + + +def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: + """Render the scenario sheets into a single branded ``.xlsx`` as bytes. Each + tuple is a Scenario name and its shaped sheet; sheet order follows the input + (the requested ``scenario_ids`` order).""" + raise NotImplementedError diff --git a/tests/infrastructure/xlsx/__init__.py b/tests/infrastructure/xlsx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/infrastructure/xlsx/test_scenario_export_workbook.py b/tests/infrastructure/xlsx/test_scenario_export_workbook.py new file mode 100644 index 000000000..d1dfaa79c --- /dev/null +++ b/tests/infrastructure/xlsx/test_scenario_export_workbook.py @@ -0,0 +1,30 @@ +"""The Scenario Export workbook renderer (ADR-0065): one branded .xlsx with one +sheet per Scenario, headers then rows, verified by re-opening the bytes.""" + +from __future__ import annotations + +from io import BytesIO + +from openpyxl import load_workbook + +from domain.scenario_export.scenario_sheet import ExportSheet +from infrastructure.xlsx.scenario_export_workbook import render_workbook + + +def test_renders_a_sheet_named_for_the_scenario_with_headers_then_rows() -> None: + # arrange — one scenario's shaped sheet. + sheet = ExportSheet( + columns=("property_id", "loft_insulation"), + rows=({"property_id": 1, "loft_insulation": 1200.0},), + ) + + # act + data = render_workbook([("Fabric first", sheet)]) + + # assert — a real workbook with a sheet named for the scenario, the column + # contract as the header row, and the data beneath it. + workbook = load_workbook(BytesIO(data)) + assert workbook.sheetnames == ["Fabric first"] + worksheet = workbook["Fabric first"] + assert [cell.value for cell in worksheet[1]] == ["property_id", "loft_insulation"] + assert [cell.value for cell in worksheet[2]] == [1, 1200.0] From 4bd90bf83661a801df181e29aaa193478b882f36 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:19:40 +0000 Subject: [PATCH 27/46] =?UTF-8?q?Render=20the=20scenario=20export=20workbo?= =?UTF-8?q?ok=20with=20a=20sheet=20per=20scenario=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../xlsx/scenario_export_workbook.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/infrastructure/xlsx/scenario_export_workbook.py b/infrastructure/xlsx/scenario_export_workbook.py index 0da8a6f4f..f86614a08 100644 --- a/infrastructure/xlsx/scenario_export_workbook.py +++ b/infrastructure/xlsx/scenario_export_workbook.py @@ -8,8 +8,11 @@ dynamic number of scenario sheets is handled natively. from __future__ import annotations +from io import BytesIO from typing import Sequence +from openpyxl import Workbook + from domain.scenario_export.scenario_sheet import ExportSheet @@ -17,4 +20,17 @@ def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: """Render the scenario sheets into a single branded ``.xlsx`` as bytes. Each tuple is a Scenario name and its shaped sheet; sheet order follows the input (the requested ``scenario_ids`` order).""" - raise NotImplementedError + workbook = Workbook() + default = workbook.active + if default is not None: + workbook.remove(default) + + for name, sheet in sheets: + worksheet = workbook.create_sheet(title=name) + worksheet.append(list(sheet.columns)) + for row in sheet.rows: + worksheet.append([row.get(column, "") for column in sheet.columns]) + + buffer = BytesIO() + workbook.save(buffer) + return buffer.getvalue() From a8c90b0d952d82e4d4a962db48b7eab1444c2cf3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:20:17 +0000 Subject: [PATCH 28/46] =?UTF-8?q?Sanitise=20and=20deduplicate=20scenario?= =?UTF-8?q?=20sheet=20names=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../xlsx/test_scenario_export_workbook.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/infrastructure/xlsx/test_scenario_export_workbook.py b/tests/infrastructure/xlsx/test_scenario_export_workbook.py index d1dfaa79c..4bf13859e 100644 --- a/tests/infrastructure/xlsx/test_scenario_export_workbook.py +++ b/tests/infrastructure/xlsx/test_scenario_export_workbook.py @@ -28,3 +28,23 @@ def test_renders_a_sheet_named_for_the_scenario_with_headers_then_rows() -> None worksheet = workbook["Fabric first"] assert [cell.value for cell in worksheet[1]] == ["property_id", "loft_insulation"] assert [cell.value for cell in worksheet[2]] == [1, 1200.0] + + +def test_sanitises_and_deduplicates_scenario_sheet_names() -> None: + # arrange — a name over Excel's 31-char limit (used twice, so it must be + # deduplicated) and one with characters Excel forbids in a sheet title. + empty = ExportSheet(columns=("property_id",), rows=()) + long_name = "A really long scenario name that exceeds Excel's limit" + illegal_name = "Bad:/\\?*[]name" + + # act + data = render_workbook( + [(long_name, empty), (long_name, empty), (illegal_name, empty)] + ) + + # assert — three distinct, Excel-legal sheet names. + names = load_workbook(BytesIO(data)).sheetnames + assert len(names) == 3 + assert len(set(names)) == 3 + assert all(len(name) <= 31 for name in names) + assert not any(ch in name for name in names for ch in r":/\?*[]") From 000ebf82bee04367456229290a3f4ac79d90eda7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:20:56 +0000 Subject: [PATCH 29/46] =?UTF-8?q?Sanitise=20and=20deduplicate=20scenario?= =?UTF-8?q?=20sheet=20names=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../xlsx/scenario_export_workbook.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/infrastructure/xlsx/scenario_export_workbook.py b/infrastructure/xlsx/scenario_export_workbook.py index f86614a08..4f971d050 100644 --- a/infrastructure/xlsx/scenario_export_workbook.py +++ b/infrastructure/xlsx/scenario_export_workbook.py @@ -8,6 +8,7 @@ dynamic number of scenario sheets is handled natively. from __future__ import annotations +import re from io import BytesIO from typing import Sequence @@ -15,6 +16,24 @@ from openpyxl import Workbook from domain.scenario_export.scenario_sheet import ExportSheet +# Excel forbids these in a sheet title and caps titles at 31 characters. +_ILLEGAL_SHEET_CHARS = re.compile(r"[:\\/?*\[\]]") +_MAX_SHEET_NAME = 31 + + +def _safe_sheet_name(name: str, used: set[str]) -> str: + """An Excel-legal, unique sheet name: illegal characters stripped, capped at + 31 chars, and suffixed on collision (two scenarios can share a name).""" + clean = _ILLEGAL_SHEET_CHARS.sub("", name or "scenario").strip() or "scenario" + clean = clean[:_MAX_SHEET_NAME] + base, index = clean, 1 + while clean in used: + suffix = f" ({index})" + clean = base[: _MAX_SHEET_NAME - len(suffix)] + suffix + index += 1 + used.add(clean) + return clean + def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: """Render the scenario sheets into a single branded ``.xlsx`` as bytes. Each @@ -25,8 +44,9 @@ def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: if default is not None: workbook.remove(default) + used_names: set[str] = set() for name, sheet in sheets: - worksheet = workbook.create_sheet(title=name) + worksheet = workbook.create_sheet(title=_safe_sheet_name(name, used_names)) worksheet.append(list(sheet.columns)) for row in sheet.rows: worksheet.append([row.get(column, "") for column in sheet.columns]) From 8f24187d03622764b844aee0a46fce07a2884862 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:22:09 +0000 Subject: [PATCH 30/46] =?UTF-8?q?Brand=20the=20export=20header=20band=20an?= =?UTF-8?q?d=20freeze=20the=20header=20row=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/xlsx/branding.py | 27 +++++++++++++++++++ .../xlsx/test_scenario_export_workbook.py | 20 ++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 infrastructure/xlsx/branding.py diff --git a/infrastructure/xlsx/branding.py b/infrastructure/xlsx/branding.py new file mode 100644 index 000000000..b53195fad --- /dev/null +++ b/infrastructure/xlsx/branding.py @@ -0,0 +1,27 @@ +"""Monday.com-style branding for the Scenario Export workbook (ADR-0065). + +A programmatic openpyxl styling pass (no template): a bold indigo header band +with white text, a frozen header row, subtle alternating row bands, and roomy +column widths. The palette lives here as the single home for brand colour — the +"no brand-colour constants anywhere" gap the review flagged. + +TODO(brand): swap the placeholder Monday-style hex for the real Domna brand +palette once design supplies it. +""" + +from __future__ import annotations + +from openpyxl.worksheet.worksheet import Worksheet + +# openpyxl colours are 8-char ARGB. Placeholder Monday-style palette. +HEADER_FILL: str = "FF6C6CFF" # indigo header band +HEADER_FONT: str = "FFFFFFFF" # white header text +ROW_BAND_FILL: str = "FFF1F1FF" # subtle alternating-row tint + +_COLUMN_WIDTH: int = 18 + + +def style_sheet(worksheet: Worksheet) -> None: + """Paint the Monday-style branding onto a populated worksheet: header band + + font, frozen header row, banded rows, and column widths.""" + raise NotImplementedError diff --git a/tests/infrastructure/xlsx/test_scenario_export_workbook.py b/tests/infrastructure/xlsx/test_scenario_export_workbook.py index 4bf13859e..81e8a3cf8 100644 --- a/tests/infrastructure/xlsx/test_scenario_export_workbook.py +++ b/tests/infrastructure/xlsx/test_scenario_export_workbook.py @@ -8,6 +8,7 @@ from io import BytesIO from openpyxl import load_workbook from domain.scenario_export.scenario_sheet import ExportSheet +from infrastructure.xlsx.branding import HEADER_FILL from infrastructure.xlsx.scenario_export_workbook import render_workbook @@ -48,3 +49,22 @@ def test_sanitises_and_deduplicates_scenario_sheet_names() -> None: assert len(set(names)) == 3 assert all(len(name) <= 31 for name in names) assert not any(ch in name for name in names for ch in r":/\?*[]") + + +def test_brands_the_header_row_and_freezes_it() -> None: + # arrange + sheet = ExportSheet( + columns=("property_id", "loft_insulation"), + rows=({"property_id": 1, "loft_insulation": 1200.0},), + ) + + # act + data = render_workbook([("Fabric first", sheet)]) + + # assert — the header is a bold, brand-filled band and the header row is + # frozen so it stays visible while scrolling. + worksheet = load_workbook(BytesIO(data))["Fabric first"] + header_cell = worksheet["A1"] + assert header_cell.font.bold is True + assert header_cell.fill.fgColor.rgb == HEADER_FILL + assert worksheet.freeze_panes == "A2" From e313f19ef21aa198f1a2296b2146cf4921354593 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:22:44 +0000 Subject: [PATCH 31/46] =?UTF-8?q?Brand=20the=20export=20header=20band=20an?= =?UTF-8?q?d=20freeze=20the=20header=20row=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- infrastructure/xlsx/branding.py | 26 ++++++++++++++++++- .../xlsx/scenario_export_workbook.py | 2 ++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/infrastructure/xlsx/branding.py b/infrastructure/xlsx/branding.py index b53195fad..b2a7f1622 100644 --- a/infrastructure/xlsx/branding.py +++ b/infrastructure/xlsx/branding.py @@ -11,6 +11,8 @@ palette once design supplies it. from __future__ import annotations +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter from openpyxl.worksheet.worksheet import Worksheet # openpyxl colours are 8-char ARGB. Placeholder Monday-style palette. @@ -24,4 +26,26 @@ _COLUMN_WIDTH: int = 18 def style_sheet(worksheet: Worksheet) -> None: """Paint the Monday-style branding onto a populated worksheet: header band + font, frozen header row, banded rows, and column widths.""" - raise NotImplementedError + header_fill = PatternFill( + start_color=HEADER_FILL, end_color=HEADER_FILL, fill_type="solid" + ) + header_font = Font(bold=True, color=HEADER_FONT) + for cell in worksheet[1]: + cell.fill = header_fill + cell.font = header_font + cell.alignment = Alignment(horizontal="left", vertical="center") + + # Keep the header visible while scrolling a long portfolio. + worksheet.freeze_panes = "A2" + + band_fill = PatternFill( + start_color=ROW_BAND_FILL, end_color=ROW_BAND_FILL, fill_type="solid" + ) + for row_index in range(3, worksheet.max_row + 1, 2): + for cell in worksheet[row_index]: + cell.fill = band_fill + + for column_index in range(1, worksheet.max_column + 1): + worksheet.column_dimensions[get_column_letter(column_index)].width = ( + _COLUMN_WIDTH + ) diff --git a/infrastructure/xlsx/scenario_export_workbook.py b/infrastructure/xlsx/scenario_export_workbook.py index 4f971d050..7dbc9bda8 100644 --- a/infrastructure/xlsx/scenario_export_workbook.py +++ b/infrastructure/xlsx/scenario_export_workbook.py @@ -15,6 +15,7 @@ from typing import Sequence from openpyxl import Workbook from domain.scenario_export.scenario_sheet import ExportSheet +from infrastructure.xlsx.branding import style_sheet # Excel forbids these in a sheet title and caps titles at 31 characters. _ILLEGAL_SHEET_CHARS = re.compile(r"[:\\/?*\[\]]") @@ -50,6 +51,7 @@ def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: worksheet.append(list(sheet.columns)) for row in sheet.rows: worksheet.append([row.get(column, "") for column in sheet.columns]) + style_sheet(worksheet) buffer = BytesIO() workbook.save(buffer) From ace3150363c113fcb07349c01068509fb630d09e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:26:59 +0000 Subject: [PATCH 32/46] =?UTF-8?q?Build=20and=20upload=20the=20sheet-per-sc?= =?UTF-8?q?enario=20export=20workbook=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- orchestration/scenario_export_orchestrator.py | 81 ++++++++++++++++ .../test_scenario_export_orchestrator.py | 95 +++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 orchestration/scenario_export_orchestrator.py create mode 100644 tests/orchestration/test_scenario_export_orchestrator.py diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py new file mode 100644 index 000000000..5accfc951 --- /dev/null +++ b/orchestration/scenario_export_orchestrator.py @@ -0,0 +1,81 @@ +"""The Scenario Export orchestrator (ADR-0065). + +Ties one export together: for each requested Scenario, read its Properties' +persisted rows, shape them into a sheet, render the sheet-per-scenario branded +workbook, upload it to the exports bucket, mint a presigned URL, and email the +requester. Best-effort email (the link is also recorded on ``sub_task.outputs``); +a selection that yields no rows for any Scenario is a recorded failure — never an +empty workbook emailed to the user. +""" + +from __future__ import annotations + +import html +import logging +from dataclasses import dataclass +from typing import Protocol, Sequence + +from domain.scenario_export.scenario_sheet import ( + PropertyScenarioData, + shape_scenario_sheet, +) +from domain.tasks.subtasks import SubTaskFailure +from infrastructure.s3.s3_client import S3Client +from infrastructure.xlsx.scenario_export_workbook import render_workbook +from repositories.email.email_sender import EmailSender + +logger = logging.getLogger(__name__) + + +class ExportRowReader(Protocol): + def rows_for( + self, + *, + portfolio_id: int, + scenario_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: ... + + +class ScenarioNames(Protocol): + def name_for(self, scenario_id: int) -> str: ... + + +@dataclass(frozen=True) +class ScenarioExportResult: + """What a completed export reports (lands on ``sub_task.outputs``).""" + + presigned_url: str + export_s3_key: str + sheet_count: int + row_count: int + + +class ScenarioExportOrchestrator: + def __init__( + self, + *, + rows: ExportRowReader, + scenarios: ScenarioNames, + exports: S3Client, + email: EmailSender, + url_ttl_seconds: int, + key_prefix: str = "scenario-exports", + ) -> None: + self._rows = rows + self._scenarios = scenarios + self._exports = exports + self._email = email + self._url_ttl_seconds = url_ttl_seconds + self._key_prefix = key_prefix + + def run( + self, + *, + portfolio_id: int, + scenario_ids: Sequence[int], + property_ids: Sequence[int], + recipient_email: str, + export_name: str, + ) -> ScenarioExportResult: + raise NotImplementedError diff --git a/tests/orchestration/test_scenario_export_orchestrator.py b/tests/orchestration/test_scenario_export_orchestrator.py new file mode 100644 index 000000000..fc918d0e8 --- /dev/null +++ b/tests/orchestration/test_scenario_export_orchestrator.py @@ -0,0 +1,95 @@ +"""The Scenario Export orchestrator (ADR-0065): read → shape → render → upload → +presign → email, best-effort, verified with in-memory fakes and a moto S3.""" + +from __future__ import annotations + +from io import BytesIO +from typing import Iterator, Optional, Sequence + +import pytest +from moto import mock_aws +from openpyxl import load_workbook + +from domain.scenario_export.scenario_sheet import ExportMeasure, PropertyScenarioData +from infrastructure.s3.s3_client import S3Client +from orchestration.scenario_export_orchestrator import ScenarioExportOrchestrator +from tests.infrastructure import make_boto_client + +EXPORTS_BUCKET = "domna-exports-dev" + + +class _FakeRows: + """Returns preset export rows per scenario id.""" + + def __init__(self, by_scenario: dict[int, list[PropertyScenarioData]]) -> None: + self._by_scenario = by_scenario + + def rows_for( + self, + *, + portfolio_id: int, + scenario_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + return self._by_scenario.get(scenario_id, []) + + +class _FakeScenarioNames: + def __init__(self, names: dict[int, str]) -> None: + self._names = names + + def name_for(self, scenario_id: int) -> str: + return self._names[scenario_id] + + +class _RecordingEmail: + def __init__(self) -> None: + self.sent: list[tuple[str, str, str, Optional[str]]] = [] + + def send( + self, *, to: str, subject: str, body: str, html_body: Optional[str] = None + ) -> None: + self.sent.append((to, subject, body, html_body)) + + +def _row(property_id: int) -> PropertyScenarioData: + return PropertyScenarioData( + property_id=property_id, + landlord_property_id=f"LP{property_id}", + measures=[ExportMeasure(measure_type="loft_insulation", estimated_cost=1200.0)], + ) + + +@pytest.fixture +def exports() -> Iterator[S3Client]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket=EXPORTS_BUCKET) + yield S3Client(boto_client, EXPORTS_BUCKET) + + +def test_builds_and_uploads_a_sheet_per_scenario_workbook(exports: S3Client) -> None: + # arrange — two scenarios, each with a modelled property. + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({5: [_row(1)], 9: [_row(1), _row(2)]}), + scenarios=_FakeScenarioNames({5: "Fabric first", 9: "Low carbon"}), + exports=exports, + email=_RecordingEmail(), + url_ttl_seconds=3600, + ) + + # act + result = orchestrator.run( + portfolio_id=100, + scenario_ids=[5, 9], + property_ids=[1, 2], + recipient_email="user@example.com", + export_name="portfolio-100", + ) + + # assert — the uploaded object is a real workbook with one sheet per scenario + # and the presigned URL points at it. + assert result.sheet_count == 2 + assert result.export_s3_key in result.presigned_url + workbook = load_workbook(BytesIO(exports.get_object(result.export_s3_key))) + assert workbook.sheetnames == ["Fabric first", "Low carbon"] From f774ca44241ad5ddb9149afce6cf3c44ce1ddb2e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:27:25 +0000 Subject: [PATCH 33/46] =?UTF-8?q?Build=20and=20upload=20the=20sheet-per-sc?= =?UTF-8?q?enario=20export=20workbook=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- orchestration/scenario_export_orchestrator.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py index 5accfc951..a45f912e7 100644 --- a/orchestration/scenario_export_orchestrator.py +++ b/orchestration/scenario_export_orchestrator.py @@ -16,6 +16,7 @@ from dataclasses import dataclass from typing import Protocol, Sequence from domain.scenario_export.scenario_sheet import ( + ExportSheet, PropertyScenarioData, shape_scenario_sheet, ) @@ -78,4 +79,26 @@ class ScenarioExportOrchestrator: recipient_email: str, export_name: str, ) -> ScenarioExportResult: - raise NotImplementedError + named_sheets: list[tuple[str, ExportSheet]] = [] + row_count = 0 + for scenario_id in scenario_ids: + rows = self._rows.rows_for( + portfolio_id=portfolio_id, + scenario_id=scenario_id, + property_ids=property_ids, + ) + sheet = shape_scenario_sheet(rows) + named_sheets.append((self._scenarios.name_for(scenario_id), sheet)) + row_count += len(sheet.rows) + + data = render_workbook(named_sheets) + key = f"{self._key_prefix}/{export_name}.xlsx" + self._exports.put_object(key, data) + url = self._exports.generate_presigned_url(key, self._url_ttl_seconds) + + return ScenarioExportResult( + presigned_url=url, + export_s3_key=key, + sheet_count=len(named_sheets), + row_count=row_count, + ) From e825b58c135cc8b11a80799e3c3a2e043e5868a3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:28:16 +0000 Subject: [PATCH 34/46] =?UTF-8?q?Email=20the=20requester=20the=20export=20?= =?UTF-8?q?link,=20best-effort=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test_scenario_export_orchestrator.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/orchestration/test_scenario_export_orchestrator.py b/tests/orchestration/test_scenario_export_orchestrator.py index fc918d0e8..17e933f81 100644 --- a/tests/orchestration/test_scenario_export_orchestrator.py +++ b/tests/orchestration/test_scenario_export_orchestrator.py @@ -93,3 +93,65 @@ def test_builds_and_uploads_a_sheet_per_scenario_workbook(exports: S3Client) -> assert result.export_s3_key in result.presigned_url workbook = load_workbook(BytesIO(exports.get_object(result.export_s3_key))) assert workbook.sheetnames == ["Fabric first", "Low carbon"] + + +def test_emails_the_requester_the_presigned_download_link(exports: S3Client) -> None: + # arrange + email = _RecordingEmail() + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({5: [_row(1)]}), + scenarios=_FakeScenarioNames({5: "Fabric first"}), + exports=exports, + email=email, + url_ttl_seconds=3600, + ) + + # act + result = orchestrator.run( + portfolio_id=100, + scenario_ids=[5], + property_ids=[1], + recipient_email="user@example.com", + export_name="portfolio-100", + ) + + # assert — the requester is emailed the link (raw URL in the plain-text + # fallback, a clean button in the HTML alternative). + assert len(email.sent) == 1 + to, _subject, body, html_body = email.sent[0] + assert to == "user@example.com" + assert result.presigned_url in body + assert html_body is not None + assert "Download" in html_body + + +class _FailingEmail: + def send( + self, *, to: str, subject: str, body: str, html_body: Optional[str] = None + ) -> None: + raise RuntimeError("smtp down") + + +def test_an_email_failure_does_not_lose_the_built_export(exports: S3Client) -> None: + # arrange — the workbook uploads fine, but email delivery blows up. + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({5: [_row(1)]}), + scenarios=_FakeScenarioNames({5: "Fabric first"}), + exports=exports, + email=_FailingEmail(), + url_ttl_seconds=3600, + ) + + # act — must not raise. + result = orchestrator.run( + portfolio_id=100, + scenario_ids=[5], + property_ids=[1], + recipient_email="user@example.com", + export_name="portfolio-100", + ) + + # assert — the export was still uploaded and its link returned (it also lands + # on sub_task.outputs, so the link isn't lost). + assert result.presigned_url + assert exports.get_object(result.export_s3_key) From 5ee8b2df505add4906d2f218404e4c4836329a91 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:28:56 +0000 Subject: [PATCH 35/46] =?UTF-8?q?Email=20the=20requester=20the=20export=20?= =?UTF-8?q?link,=20best-effort=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- orchestration/scenario_export_orchestrator.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py index a45f912e7..4166d27c0 100644 --- a/orchestration/scenario_export_orchestrator.py +++ b/orchestration/scenario_export_orchestrator.py @@ -96,9 +96,60 @@ class ScenarioExportOrchestrator: self._exports.put_object(key, data) url = self._exports.generate_presigned_url(key, self._url_ttl_seconds) + subject, body, html_body = self._compose_email( + url=url, sheets=len(named_sheets), rows=row_count + ) + try: + self._email.send( + to=recipient_email, subject=subject, body=body, html_body=html_body + ) + logger.info("scenario_export: emailed %s", recipient_email) + except Exception: + # Best-effort delivery (ADR-0065): the link is also written to + # sub_task.outputs, so an email/transport failure must not lose an + # export that was already built and uploaded. + logger.warning( + "scenario_export: email delivery to %s failed; the link is still " + "recorded on sub_task.outputs", + recipient_email, + exc_info=True, + ) + return ScenarioExportResult( presigned_url=url, export_s3_key=key, sheet_count=len(named_sheets), row_count=row_count, ) + + def _compose_email( + self, *, url: str, sheets: int, rows: int + ) -> tuple[str, str, str]: + """The delivery email (ADR-0059/0065): a subject, a plain-text body + carrying the raw link, and an HTML alternative with a clean download + button so the long presigned URL isn't the visible content.""" + minutes = self._url_ttl_seconds // 60 + sheet_noun = "scenario" if sheets == 1 else "scenarios" + subject = f"Your scenario export is ready ({sheets} {sheet_noun})" + body = ( + "Your scenario export is ready.\n\n" + f"It covers {sheets} {sheet_noun} across {rows} property row(s).\n\n" + f"Download it here (link valid for {minutes} minutes):\n{url}\n" + ) + safe_url = html.escape(url, quote=True) + html_body = ( + '
' + "

Your scenario export is ready.

" + f"

It covers {sheets} {sheet_noun} across " + f"{rows} property row(s).

" + '

' + f'Download export

' + f'

This link expires in {minutes} ' + "minutes. If the button doesn't work, paste this URL into your browser:" + f'
{safe_url}

' + "
" + ) + return subject, body, html_body From 4220ce4bf04fd65381bba884563b3fc402ebd476 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:29:28 +0000 Subject: [PATCH 36/46] =?UTF-8?q?Record=20a=20failure=20when=20the=20selec?= =?UTF-8?q?tion=20yields=20no=20export=20rows=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test_scenario_export_orchestrator.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/orchestration/test_scenario_export_orchestrator.py b/tests/orchestration/test_scenario_export_orchestrator.py index 17e933f81..86ce864ea 100644 --- a/tests/orchestration/test_scenario_export_orchestrator.py +++ b/tests/orchestration/test_scenario_export_orchestrator.py @@ -11,6 +11,7 @@ from moto import mock_aws from openpyxl import load_workbook from domain.scenario_export.scenario_sheet import ExportMeasure, PropertyScenarioData +from domain.tasks.subtasks import SubTaskFailure from infrastructure.s3.s3_client import S3Client from orchestration.scenario_export_orchestrator import ScenarioExportOrchestrator from tests.infrastructure import make_boto_client @@ -155,3 +156,29 @@ def test_an_email_failure_does_not_lose_the_built_export(exports: S3Client) -> N # on sub_task.outputs, so the link isn't lost). assert result.presigned_url assert exports.get_object(result.export_s3_key) + + +def test_a_selection_that_yields_no_rows_is_a_recorded_failure( + exports: S3Client, +) -> None: + # arrange — no property has a default Plan under the scenario (model A), so + # the repo returns no rows for it. + email = _RecordingEmail() + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({}), + scenarios=_FakeScenarioNames({5: "Fabric first"}), + exports=exports, + email=email, + url_ttl_seconds=3600, + ) + + # act / assert — a recorded failure, never an empty workbook emailed. + with pytest.raises(SubTaskFailure): + orchestrator.run( + portfolio_id=100, + scenario_ids=[5], + property_ids=[1], + recipient_email="user@example.com", + export_name="portfolio-100", + ) + assert email.sent == [] From 42201e42c51efb44ba8d5ce46ca171be2f015b0a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:29:45 +0000 Subject: [PATCH 37/46] =?UTF-8?q?Record=20a=20failure=20when=20the=20selec?= =?UTF-8?q?tion=20yields=20no=20export=20rows=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- orchestration/scenario_export_orchestrator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py index 4166d27c0..282751eb4 100644 --- a/orchestration/scenario_export_orchestrator.py +++ b/orchestration/scenario_export_orchestrator.py @@ -91,6 +91,18 @@ class ScenarioExportOrchestrator: named_sheets.append((self._scenarios.name_for(scenario_id), sheet)) row_count += len(sheet.rows) + if row_count == 0: + # No Property had a default Plan under any requested Scenario (model + # A) — a recorded failure, never an empty workbook emailed (ADR-0065). + raise SubTaskFailure( + "the scenario export selection produced no exportable rows", + details={ + "portfolio_id": portfolio_id, + "scenario_ids": list(scenario_ids), + "property_count": len(property_ids), + }, + ) + data = render_workbook(named_sheets) key = f"{self._key_prefix}/{export_name}.xlsx" self._exports.put_object(key, data) From ddc19949840a2f0df7424449ed6056368b09669a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:32:14 +0000 Subject: [PATCH 38/46] =?UTF-8?q?Resolve=20export=20property=20ids=20with?= =?UTF-8?q?=20explicit=20ids=20overriding=20filters=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/exports/__init__.py | 0 backend/app/exports/export_tasks.py | 99 +++++++++++++++++++ tests/backend/app/exports/__init__.py | 0 .../backend/app/exports/test_export_tasks.py | 49 +++++++++ 4 files changed, 148 insertions(+) create mode 100644 backend/app/exports/__init__.py create mode 100644 backend/app/exports/export_tasks.py create mode 100644 tests/backend/app/exports/__init__.py create mode 100644 tests/backend/app/exports/test_export_tasks.py diff --git a/backend/app/exports/__init__.py b/backend/app/exports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/app/exports/export_tasks.py b/backend/app/exports/export_tasks.py new file mode 100644 index 000000000..fbe92e342 --- /dev/null +++ b/backend/app/exports/export_tasks.py @@ -0,0 +1,99 @@ +"""The Scenario Export route's view of the app-owned task's sub_task (ADR-0065). + +Parameterised SQL rather than the SQLModel mirrors: importing the +``infrastructure.postgres`` mirror of ``sub_task`` into the FastAPI process +double-registers the table and crashes at import (the same constraint the +Modelling Run distributor and the bulk-download route work around). +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, cast +from uuid import UUID + +from sqlalchemy import text +from sqlmodel import Session + +from backend.app.modelling.property_filters import ( + PropertyGroupFilters, + resolve_filtered_property_ids, +) + + +class ScenarioExportTasks: + def __init__(self, session: Session) -> None: + self._session = session + + def read_selection_config(self, task_id: UUID) -> dict[str, Any]: + """The task's selection recipe, read from the FE-owned ``task.inputs`` + JSON (ADR-0065) — ``{portfolio_id, scenario_ids, filters?, property_ids?, + recipient_email?}``. Empty dict when the task has no inputs.""" + row = ( + self._session.connection() + .execute( + text("SELECT inputs FROM tasks WHERE id = :task_id"), + {"task_id": task_id}, + ) + .first() + ) + if row is None or row[0] is None: + return {} + parsed: Any = json.loads(row[0]) + return cast(dict[str, Any], parsed) if isinstance(parsed, dict) else {} + + def already_distributed(self, task_id: UUID) -> bool: + """Whether the task already has a sub_task — the 409 guard against a + double-submit re-triggering the same export.""" + row = ( + self._session.connection() + .execute( + text("SELECT 1 FROM sub_task WHERE task_id = :task_id LIMIT 1"), + {"task_id": task_id}, + ) + .first() + ) + return row is not None + + def resolve_property_ids( + self, + *, + portfolio_id: int, + filters: PropertyGroupFilters, + property_ids: list[int], + ) -> list[int]: + """Resolve the selection to the distinct internal ``property.id`` set the + export is built from (ADR-0065). An explicit ``property_ids`` list is an + **override**, not a co-filter: when present it is used as given; + otherwise the ``filters`` are resolved against the portfolio through the + shared Modelling Run resolver (ADR-0056).""" + raise NotImplementedError + + def create_export_subtask( + self, task_id: UUID, subtask_id: UUID, recipe: dict[str, Any] + ) -> bool: + """Pre-create the single ``waiting`` sub_task under the app-owned task, + pinning the recipe (resolved ``property_ids``, ``scenario_ids``, + ``recipient_email``, ``export_name``) onto its inputs (ADR-0055/0065). + + The insert is conditional on the task having no sub_task yet, so the DB + arbitrates a double-submit race. Returns whether this call created it.""" + now = datetime.now(timezone.utc) + result = self._session.connection().execute( + text( + "INSERT INTO sub_task (id, task_id, status, inputs, updated_at)" + " SELECT :id, :task_id, 'waiting', :inputs, :updated_at" + " WHERE NOT EXISTS (" + " SELECT 1 FROM sub_task WHERE task_id = :task_id" + " )" + ), + { + "id": subtask_id, + "task_id": task_id, + "inputs": json.dumps(recipe), + "updated_at": now, + }, + ) + self._session.commit() + return result.rowcount == 1 diff --git a/tests/backend/app/exports/__init__.py b/tests/backend/app/exports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/backend/app/exports/test_export_tasks.py b/tests/backend/app/exports/test_export_tasks.py new file mode 100644 index 000000000..7026c7bc6 --- /dev/null +++ b/tests/backend/app/exports/test_export_tasks.py @@ -0,0 +1,49 @@ +"""ScenarioExportTasks property-id resolution (ADR-0065): an explicit +property_ids list overrides the filters; otherwise the shared Modelling Run +filter resolver (ADR-0056) resolves the portfolio's filtered set.""" + +from __future__ import annotations + +from sqlalchemy import Engine +from sqlmodel import Session + +from backend.app.exports.export_tasks import ScenarioExportTasks +from backend.app.modelling.property_filters import PropertyGroupFilters +from infrastructure.postgres.property_table import PropertyRow # registers `property` + + +def test_explicit_property_ids_override_the_filters(db_engine: Engine) -> None: + # arrange — filters that match nothing, but an explicit hand-picked list. + with Session(db_engine) as session: + # act + result = ScenarioExportTasks(session).resolve_property_ids( + portfolio_id=100, + filters=PropertyGroupFilters(postcodes=["ZZ1 1ZZ"]), + property_ids=[3, 1, 2, 1], + ) + + # assert — the hand-picked ids win (deduplicated, sorted); filters ignored. + assert result == [1, 2, 3] + + +def test_resolves_from_filters_when_no_explicit_ids(db_engine: Engine) -> None: + # arrange — two properties in the portfolio, one matching the postcode filter. + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, postcode="AB1 2CD", landlord_property_id="LP1") + ) + session.add( + PropertyRow(id=2, portfolio_id=100, postcode="XY9 9ZZ", landlord_property_id="LP2") + ) + session.commit() + + # act + with Session(db_engine) as session: + result = ScenarioExportTasks(session).resolve_property_ids( + portfolio_id=100, + filters=PropertyGroupFilters(postcodes=["AB1 2CD"]), + property_ids=[], + ) + + # assert — only the property matching the filter is resolved. + assert result == [1] From 274d71599e8a2d9ce3371c3fe9796fd6a3dbecb2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:32:39 +0000 Subject: [PATCH 39/46] =?UTF-8?q?Resolve=20export=20property=20ids=20with?= =?UTF-8?q?=20explicit=20ids=20overriding=20filters=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/exports/export_tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/app/exports/export_tasks.py b/backend/app/exports/export_tasks.py index fbe92e342..c323058e7 100644 --- a/backend/app/exports/export_tasks.py +++ b/backend/app/exports/export_tasks.py @@ -68,7 +68,12 @@ class ScenarioExportTasks: **override**, not a co-filter: when present it is used as given; otherwise the ``filters`` are resolved against the portfolio through the shared Modelling Run resolver (ADR-0056).""" - raise NotImplementedError + if property_ids: + return sorted(set(property_ids)) + resolved = resolve_filtered_property_ids( + self._session, portfolio_id, filters + ) + return sorted({filtered.property_id for filtered in resolved}) def create_export_subtask( self, task_id: UUID, subtask_id: UUID, recipe: dict[str, Any] From 8bd9c564dce851b3d0d74e5e721308c2fce541e3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:36:14 +0000 Subject: [PATCH 40/46] =?UTF-8?q?Add=20the=20scenario=20export=20route,=20?= =?UTF-8?q?worker=20handler=20and=20SQS=20trigger=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires POST /v1/exports/scenario -> tasks.inputs recipe -> pinned sub_task -> ARA_EXPORT_SQS_URL -> ara_export Lambda -> orchestrator. Route resolution and the trigger body are covered by tests; the Lambda handler mirrors bulk_document_download. Co-Authored-By: Claude Opus 4.8 (1M context) --- applications/ara_export/__init__.py | 0 .../ara_export/ara_export_trigger_body.py | 19 ++ applications/ara_export/handler.py | 121 +++++++++++++ backend/app/config.py | 1 + backend/app/exports/router.py | 170 ++++++++++++++++++ backend/app/exports/schemas.py | 12 ++ backend/app/main.py | 2 + tests/applications/ara_export/__init__.py | 0 .../test_ara_export_trigger_body.py | 24 +++ 9 files changed, 349 insertions(+) create mode 100644 applications/ara_export/__init__.py create mode 100644 applications/ara_export/ara_export_trigger_body.py create mode 100644 applications/ara_export/handler.py create mode 100644 backend/app/exports/router.py create mode 100644 backend/app/exports/schemas.py create mode 100644 tests/applications/ara_export/__init__.py create mode 100644 tests/applications/ara_export/test_ara_export_trigger_body.py diff --git a/applications/ara_export/__init__.py b/applications/ara_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/applications/ara_export/ara_export_trigger_body.py b/applications/ara_export/ara_export_trigger_body.py new file mode 100644 index 000000000..cf12f13be --- /dev/null +++ b/applications/ara_export/ara_export_trigger_body.py @@ -0,0 +1,19 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class AraExportTriggerBody(BaseModel): + """SQS trigger for the ara_export Lambda (ADR-0055/0065). + + Attach mode: the FastAPI route created the Task + SubTask and pinned the + recipe (``portfolio_id``, ``scenario_ids``, ``property_ids``, + ``recipient_email``, ``export_name``) onto the SubTask's ``inputs``. The + message therefore carries only the identifiers — the (potentially large) + property set never travels through the 256 KB SQS body. + """ + + model_config = ConfigDict(extra="allow") + + task_id: UUID + subtask_id: UUID diff --git a/applications/ara_export/handler.py b/applications/ara_export/handler.py new file mode 100644 index 000000000..9e78e2a7b --- /dev/null +++ b/applications/ara_export/handler.py @@ -0,0 +1,121 @@ +"""ara_export Lambda (ADR-0065). + +Attach-mode (ADR-0055) handler. The FastAPI route created the app-owned Task + +SubTask and pinned the recipe onto the SubTask's ``inputs``; this reads that +recipe, builds the sheet-per-scenario branded workbook through the orchestrator, +and returns the result (presigned URL + counts), which ``TaskOrchestrator`` +records on ``sub_task.outputs``. A selection that yields no rows raises +``SubTaskFailure`` from the orchestrator — recorded, not retried. + +PostgresConfig-only (POSTGRES_*), like the bulk_document_download Lambda. The +workbook is written to the dedicated ``DOCUMENT_EXPORTS_BUCKET``; SES SMTP +settings arrive as env vars. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +import boto3 +from sqlalchemy import text +from sqlmodel import Session + +from applications.ara_export.ara_export_trigger_body import AraExportTriggerBody +from domain.tasks.tasks import Source +from infrastructure.email.ses_smtp_email_sender import SesSmtpEmailSender +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine, make_session +from infrastructure.s3.s3_client import S3Client +from orchestration.scenario_export_orchestrator import ScenarioExportOrchestrator +from repositories.scenario_export.scenario_export_postgres_repository import ( + ScenarioExportPostgresRepository, +) +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from utilities.aws_lambda.task_handler import task_handler + +logging.getLogger().setLevel(logging.INFO) +logger = logging.getLogger(__name__) + +# The export link is valid for 60 minutes (ADR-0065). +_URL_TTL_SECONDS = 3600 + + +class _ScenarioNames: + """Scenario id → display name, read from the ``scenario`` table (the domain + Scenario aggregate does not carry the name).""" + + def __init__(self, session: Session, scenario_ids: list[int]) -> None: + rows = ( + session.connection() + .execute( + text("SELECT id, name FROM scenario WHERE id = ANY(:ids)"), + {"ids": scenario_ids}, + ) + .all() + ) + self._names: dict[int, str] = { + row[0]: (row[1] or f"scenario_{row[0]}") for row in rows + } + + def name_for(self, scenario_id: int) -> str: + return self._names.get(scenario_id, f"scenario_{scenario_id}") + + +def _email_sender() -> SesSmtpEmailSender: + return SesSmtpEmailSender( + host=os.environ["SES_SMTP_HOST"], + port=int(os.environ["SES_SMTP_PORT"]), + username=os.environ["SES_SMTP_USERNAME"], + password=os.environ["SES_SMTP_PASSWORD"], + from_address=os.environ["SES_SMTP_FROM_ADDRESS"], + ) + + +@task_handler(task_source="ara_export", source=Source.PORTFOLIO) +def handler(body: dict[str, Any], context: Any) -> dict[str, Any]: + trigger = AraExportTriggerBody.model_validate(body) + engine = make_engine(PostgresConfig.from_env(os.environ)) + session = make_session(engine) + try: + subtask = SubTaskPostgresRepository(session).get(trigger.subtask_id) + recipe: dict[str, Any] = subtask.inputs or {} + portfolio_id = int(recipe["portfolio_id"]) + scenario_ids: list[int] = [int(x) for x in recipe["scenario_ids"]] + property_ids: list[int] = [int(x) for x in recipe["property_ids"]] + recipient_email = str(recipe["recipient_email"]) + export_name = str(recipe["export_name"]) + + logger.info( + "ara_export: starting export for %d properties across %d scenario(s) " + "(subtask=%s)", + len(property_ids), + len(scenario_ids), + trigger.subtask_id, + ) + + boto_s3: Any = boto3.client("s3") # pyright: ignore[reportUnknownMemberType] + orchestrator = ScenarioExportOrchestrator( + rows=ScenarioExportPostgresRepository(session), + scenarios=_ScenarioNames(session, scenario_ids), + exports=S3Client(boto_s3, os.environ["DOCUMENT_EXPORTS_BUCKET"]), + email=_email_sender(), + url_ttl_seconds=_URL_TTL_SECONDS, + ) + result = orchestrator.run( + portfolio_id=portfolio_id, + scenario_ids=scenario_ids, + property_ids=property_ids, + recipient_email=recipient_email, + export_name=export_name, + ) + finally: + session.close() + + return { + "presigned_url": result.presigned_url, + "export_s3_key": result.export_s3_key, + "sheet_count": result.sheet_count, + "row_count": result.row_count, + } diff --git a/backend/app/config.py b/backend/app/config.py index 8f0d551f1..c5ec42948 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -47,6 +47,7 @@ class Settings(BaseSettings): FINALISER_SQS_URL: str = "changeme" MODELLING_E2E_SQS_URL: str = "changeme" BULK_DOCUMENT_DOWNLOAD_SQS_URL: str = "changeme" + ARA_EXPORT_SQS_URL: str = "changeme" # Third parties EPC_AUTH_TOKEN: str = "changeme" diff --git a/backend/app/exports/router.py b/backend/app/exports/router.py new file mode 100644 index 000000000..e4d99b9d0 --- /dev/null +++ b/backend/app/exports/router.py @@ -0,0 +1,170 @@ +"""Scenario Export trigger: POST /v1/exports/scenario (ADR-0065). + +Resolves the authenticated requester's email, reads the FE-written selection +recipe from ``task.inputs``, resolves + caps the property selection (an explicit +``property_ids`` list overrides the filters; otherwise the shared Modelling Run +filter resolver runs), pins the resolved recipe onto a single pre-created +sub_task under the app-owned task, and enqueues one message to the ara_export +worker. Never builds the workbook synchronously — the worker emails the link. +""" + +import json +from collections.abc import Callable, Iterator +from typing import Any, cast +from uuid import uuid4 + +import boto3 +from fastapi import APIRouter, Depends, HTTPException +from sqlmodel import Session + +from backend.app.config import get_settings +from backend.app.db.connection import db_engine +from backend.app.dependencies import validate_jwt_token, validate_token +from backend.app.exports.export_tasks import ScenarioExportTasks +from backend.app.exports.schemas import ScenarioExportRequest +from backend.app.modelling.property_filters import PropertyGroupFilters + +# The selection cap (ADR-0065) — rejected at the trigger so an oversized export +# never reaches the worker. Configurable ceiling; the JSON-column mechanism has +# no limit of its own. +MAX_PROPERTIES = 100_000 + +MessageSender = Callable[[str], None] + + +def get_session() -> Iterator[Session]: + with Session(db_engine) as session: + yield session + + +def get_message_sender() -> MessageSender: + settings = get_settings() + client: Any = cast(Any, boto3.client("sqs", settings.AWS_DEFAULT_REGION)) # pyright: ignore[reportUnknownMemberType] + queue_url = settings.ARA_EXPORT_SQS_URL + + def send(body: str) -> None: + client.send_message(QueueUrl=queue_url, MessageBody=body) + + return send + + +def get_requesting_user_email(user: Any = Depends(validate_jwt_token)) -> str: + """The authenticated requester's email — the export recipient (ADR-0059).""" + email: Any = getattr(user, "email", None) + if email is None and isinstance(user, dict): + email = cast(dict[str, Any], user).get("email") + if not email and get_settings().ENVIRONMENT == "local": + email = "local-dev@example.com" + if not email: + raise HTTPException( + status_code=400, + detail="Could not determine the requesting user's email address.", + ) + return str(email) + + +router = APIRouter( + prefix="/exports", + tags=["exports"], + dependencies=[Depends(validate_token)], +) + + +@router.post("/scenario", status_code=202) +async def scenario_export( + body: ScenarioExportRequest, + session: Session = Depends(get_session), + send_message: MessageSender = Depends(get_message_sender), + recipient_email: str = Depends(get_requesting_user_email), +) -> dict[str, str]: + tasks = ScenarioExportTasks(session) + if tasks.already_distributed(body.task_id): + raise HTTPException( + status_code=409, + detail=( + f"Task {body.task_id} already has a sub_task — an export has " + "already been started for it." + ), + ) + + config = tasks.read_selection_config(body.task_id) + portfolio_id: Any = config.get("portfolio_id") + raw_scenario_ids: Any = config.get("scenario_ids") or [] + raw_property_ids: Any = config.get("property_ids") or [] + raw_filters: Any = config.get("filters") or {} + + if not isinstance(portfolio_id, int): + raise HTTPException( + status_code=400, detail="task.inputs must provide an integer portfolio_id." + ) + if not _is_int_list(raw_scenario_ids) or not raw_scenario_ids: + raise HTTPException( + status_code=400, + detail="task.inputs scenario_ids must be a non-empty list of integers.", + ) + if not _is_int_list(raw_property_ids): + raise HTTPException( + status_code=400, + detail="task.inputs property_ids must be a list of integers.", + ) + if not isinstance(raw_filters, dict): + raise HTTPException( + status_code=400, detail="task.inputs filters must be an object." + ) + + scenario_ids: list[int] = cast(list[int], raw_scenario_ids) + property_ids: list[int] = cast(list[int], raw_property_ids) + try: + filters = PropertyGroupFilters(**cast(dict[str, Any], raw_filters)) + except Exception as exc: # pragma: no cover - pydantic validation detail + raise HTTPException( + status_code=400, detail=f"task.inputs filters are invalid: {exc}" + ) from exc + + resolved_property_ids = tasks.resolve_property_ids( + portfolio_id=portfolio_id, filters=filters, property_ids=property_ids + ) + if not resolved_property_ids: + raise HTTPException( + status_code=400, + detail="The selection resolves to no properties.", + ) + if len(resolved_property_ids) > MAX_PROPERTIES: + raise HTTPException( + status_code=400, + detail=( + f"The selection has {len(resolved_property_ids)} properties, over " + f"the {MAX_PROPERTIES} limit — narrow your selection." + ), + ) + + subtask_id = uuid4() + recipe = { + "portfolio_id": portfolio_id, + "scenario_ids": scenario_ids, + "property_ids": resolved_property_ids, + "recipient_email": recipient_email, + "export_name": f"portfolio-{portfolio_id}-{body.task_id}", + } + created = tasks.create_export_subtask(body.task_id, subtask_id, recipe) + if not created: + raise HTTPException( + status_code=409, + detail=( + f"Task {body.task_id} already has a sub_task — an export has " + "already been started for it." + ), + ) + + send_message( + json.dumps({"task_id": str(body.task_id), "subtask_id": str(subtask_id)}) + ) + return {"message": "Scenario export started"} + + +def _is_int_list(value: Any) -> bool: + # bool is an int subclass; exclude it so a stray True/False isn't a valid id. + return isinstance(value, list) and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast(list[Any], value) + ) diff --git a/backend/app/exports/schemas.py b/backend/app/exports/schemas.py new file mode 100644 index 000000000..bd3554d07 --- /dev/null +++ b/backend/app/exports/schemas.py @@ -0,0 +1,12 @@ +from uuid import UUID + +from pydantic import BaseModel + + +class ScenarioExportRequest(BaseModel): + """The Scenario Export trigger body (ADR-0065). Carries only the app-owned + ``task_id``: the FE wrote the selection recipe (portfolio, scenarios, + filters, optional property_ids) into ``task.inputs``, so the (potentially + large) selection never travels in the HTTP body.""" + + task_id: UUID diff --git a/backend/app/main.py b/backend/app/main.py index 270917f63..2060d0604 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -12,6 +12,7 @@ from backend.app.plan import router as plan_router from backend.app.tasks import router as tasks_router from backend.app.bulk_uploads import router as bulk_uploads_router from backend.app.documents import router as documents_router +from backend.app.exports import router as exports_router from backend.app.modelling import router as modelling_router from backend.app.dependencies import validate_api_key from backend.app.config import get_settings @@ -67,6 +68,7 @@ app.include_router(whlg_router.router, prefix="/v1") app.include_router(bulk_uploads_router.router, prefix="/v1") app.include_router(modelling_router.router, prefix="/v1") app.include_router(documents_router.router, prefix="/v1") +app.include_router(exports_router.router, prefix="/v1") if get_settings().ENVIRONMENT == "local": from backend.app.local import router as local_router diff --git a/tests/applications/ara_export/__init__.py b/tests/applications/ara_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/applications/ara_export/test_ara_export_trigger_body.py b/tests/applications/ara_export/test_ara_export_trigger_body.py new file mode 100644 index 000000000..74c9cbfc2 --- /dev/null +++ b/tests/applications/ara_export/test_ara_export_trigger_body.py @@ -0,0 +1,24 @@ +"""The ara_export SQS trigger body (ADR-0065): carries only the task/subtask +identifiers (the selection rides sub_task.inputs), tolerating extra fields.""" + +from __future__ import annotations + +from uuid import UUID + +from applications.ara_export.ara_export_trigger_body import AraExportTriggerBody + + +def test_parses_the_task_and_subtask_ids_and_tolerates_extra_fields() -> None: + # arrange — a message body with the two ids and an extra AWS-added field. + raw = { + "task_id": "11111111-1111-1111-1111-111111111111", + "subtask_id": "22222222-2222-2222-2222-222222222222", + "Records": "ignored", + } + + # act + body = AraExportTriggerBody.model_validate(raw) + + # assert — the ids parse as UUIDs; the extra field is tolerated, not fatal. + assert body.task_id == UUID("11111111-1111-1111-1111-111111111111") + assert body.subtask_id == UUID("22222222-2222-2222-2222-222222222222") From 30c7133b9ae653e20c7dc5ccbe6263d18e95ca3e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 09:38:07 +0000 Subject: [PATCH 41/46] =?UTF-8?q?Add=20ara=5Fexport=20Lambda=20+=20SQS=20t?= =?UTF-8?q?erraform=20module=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained root module mirroring bulk_document_download (4GB memory per ADR-0065, exports-bucket-only IAM). Cross-module wiring (shared ECR/state, fast-api remote_state + ARA_EXPORT_SQS_URL env, CI image build) still needs a terraform plan review — unverifiable in this environment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../terraform/lambda/ara_export/main.tf | 84 +++++++++++++++++++ .../terraform/lambda/ara_export/outputs.tf | 9 ++ .../terraform/lambda/ara_export/provider.tf | 20 +++++ .../terraform/lambda/ara_export/variables.tf | 61 ++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 deployment/terraform/lambda/ara_export/main.tf create mode 100644 deployment/terraform/lambda/ara_export/outputs.tf create mode 100644 deployment/terraform/lambda/ara_export/provider.tf create mode 100644 deployment/terraform/lambda/ara_export/variables.tf diff --git a/deployment/terraform/lambda/ara_export/main.tf b/deployment/terraform/lambda/ara_export/main.tf new file mode 100644 index 000000000..04978e69e --- /dev/null +++ b/deployment/terraform/lambda/ara_export/main.tf @@ -0,0 +1,84 @@ +############################################ +# ara_export Lambda (ADR-0065) — builds one branded sheet-per-scenario .xlsx +# from persisted modelling data, uploads it to the exports bucket, and emails a +# presigned link. Reads only the DB (no source-bucket reads, unlike +# bulk_document_download), so the role needs only exports-bucket write/presign. +############################################ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +data "aws_secretsmanager_secret_version" "ses_smtp" { + secret_id = "${var.stage}/ses/smtp_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) + ses_smtp = jsondecode(data.aws_secretsmanager_secret_version.ses_smtp.secret_string) + + document_exports_bucket = "retrofit-document-exports-${var.stage}" +} + +############################################ +# Lambda + SQS queue + trigger +############################################ +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = var.lambda_name + stage = var.stage + + image_uri = local.image_uri + + reserved_concurrent_executions = var.reserved_concurrent_executions + + batch_size = var.batch_size + maximum_concurrency = var.maximum_concurrency + + timeout = 900 + # Normal-mode openpyxl holds the workbook in memory (ADR-0065); 4 GB covers the + # 100k-property ceiling. Raise here (then switch to write_only) if it OOMs. + memory_size = 4096 + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_HOST = var.db_host + POSTGRES_DATABASE = var.db_name + POSTGRES_PORT = var.db_port + + DOCUMENT_EXPORTS_BUCKET = local.document_exports_bucket + + # SES SMTP — IAM-user credentials sourced from Secrets Manager (modules/ses). + SES_SMTP_HOST = "email-smtp.eu-west-2.amazonaws.com" + SES_SMTP_PORT = "587" + SES_SMTP_USERNAME = local.ses_smtp.username + SES_SMTP_PASSWORD = local.ses_smtp.password + SES_SMTP_FROM_ADDRESS = var.ses_from_address + } +} + +############################################ +# IAM: write + presign-read the workbook on the dedicated exports bucket. +# GetObject is required so the Lambda-signed presigned GET URL resolves. +############################################ +module "s3_exports" { + source = "../../modules/s3_iam_policy" + + policy_name = "AraExportS3Exports-${var.stage}" + policy_description = "Allow ara_export Lambda to write and presign-read Scenario Exports on the exports bucket" + bucket_arns = ["arn:aws:s3:::${local.document_exports_bucket}"] + actions = ["s3:PutObject", "s3:GetObject"] + resource_paths = ["/*"] +} + +resource "aws_iam_role_policy_attachment" "s3_exports" { + role = module.lambda.role_name + policy_arn = module.s3_exports.policy_arn +} + +# NOTE: no ses:* on the role — the handler sends over SMTP with the SES IAM-user +# credentials injected above. diff --git a/deployment/terraform/lambda/ara_export/outputs.tf b/deployment/terraform/lambda/ara_export/outputs.tf new file mode 100644 index 000000000..08e258638 --- /dev/null +++ b/deployment/terraform/lambda/ara_export/outputs.tf @@ -0,0 +1,9 @@ +output "ara_export_queue_url" { + value = module.lambda.queue_url + description = "URL of the ara-export SQS queue (FastAPI enqueues jobs here)" +} + +output "ara_export_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the ara-export SQS queue" +} diff --git a/deployment/terraform/lambda/ara_export/provider.tf b/deployment/terraform/lambda/ara_export/provider.tf new file mode 100644 index 000000000..ec65a064b --- /dev/null +++ b/deployment/terraform/lambda/ara_export/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "ara-export-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "eu-west-2" +} diff --git a/deployment/terraform/lambda/ara_export/variables.tf b/deployment/terraform/lambda/ara_export/variables.tf new file mode 100644 index 000000000..e8425d492 --- /dev/null +++ b/deployment/terraform/lambda/ara_export/variables.tf @@ -0,0 +1,61 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "reserved_concurrent_executions" { + type = number + default = -1 + description = "Reserved concurrency for the Lambda. -1 = unreserved." +} + +variable "maximum_concurrency" { + type = number + default = 5 + description = "Maximum concurrent Lambda invocations from the SQS trigger." +} + +variable "batch_size" { + type = number + default = 1 +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} + +variable "ses_from_address" { + type = string + description = "Verified SES sender for Scenario Export notifications (ADR-0059)." + default = "noreply@domna.homes" +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} From a820274d4fc9d70883c8a0adbad8479cf605cd54 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 16 Jul 2026 01:26:32 +0000 Subject: [PATCH 42/46] ARA export: newest plan per (property, scenario) + uprn-matched Effective-EPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes so a multi-scenario export over a PasHub-fetched portfolio is correct: 1. Plan selection: take the NEWEST plan per (property, scenario), not the is_default one. is_default is one-per-property (not per-scenario), so it cannot scope a multi-scenario export — a property's default sits under a single scenario, leaving every other scenario with zero rows. _ROWS_SQL and _MEASURES_SQL now DISTINCT ON (property_id) ORDER BY created_at DESC, id DESC. 2. Effective-EPC descriptive block: match the header facts + current performance by UPRN (the latest epc_property for the source), because the PasHub re-fetch lands records with property_id = NULL — a property_id join silently fell back to a stale gov-EPC cert (wrong 2013-2025 lodgement dates / int property_type) or nothing. Lodgement date coalesces registration -> completion -> inspection (PasHub surveys carry only inspection_date). Descriptive prose elements stay on the property_id gov-EPC fallback (a survey lodges structured fields, not gov-EPC prose). Result on portfolio 838 / scenarios 1303+1304: both sheets populate 205 rows; property_type text on 204/205; lodgement dates 2026 on 200/205. Repository tests updated (uprn fixtures, newest-plan selection + a newest-wins case); 22 export tests pass, pyright clean. --- .../scenario_export_postgres_repository.py | 85 ++++++++++++------- ...est_scenario_export_postgres_repository.py | 66 +++++++++++--- 2 files changed, 110 insertions(+), 41 deletions(-) diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 4029b41eb..a2674b20f 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -11,50 +11,69 @@ from repositories.scenario_export.scenario_export_repository import ( ScenarioExportRepository, ) -# The default Plan per (scenario, property), scoped to the requested Properties. -# The JOIN to `plan` is the model-A scenario-scoping (ADR-0065): a Property with -# no default Plan under the Scenario yields no row and drops out. +# The NEWEST Plan per (property, scenario), scoped to the requested Properties. +# `is_default` is one-per-property (not per-scenario), so it cannot scope a +# multi-scenario export — the freshest Plan under the Scenario (created_at DESC, +# id tiebreak) is the export's selection. A Property with no Plan under the +# Scenario yields no row and drops out. _ROWS_SQL = text( "SELECT p.id, p.landlord_property_id, p.uprn, p.address, p.postcode," - " pl.post_sap_points, pl.post_epc_rating, pl.cost_of_works," - " pl.contingency_cost, pl.co2_savings, pl.energy_bill_savings," - " pl.energy_consumption_savings, pl.valuation_increase" + " np.post_sap_points, np.post_epc_rating, np.cost_of_works," + " np.contingency_cost, np.co2_savings, np.energy_bill_savings," + " np.energy_consumption_savings, np.valuation_increase" " FROM property p" - " JOIN plan pl ON pl.property_id = p.id" + " JOIN (" + " SELECT DISTINCT ON (property_id) property_id, post_sap_points," + " post_epc_rating, cost_of_works, contingency_cost, co2_savings," + " energy_bill_savings, energy_consumption_savings, valuation_increase" + " FROM plan" + " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" + " AND property_id = ANY(:property_ids)" + " ORDER BY property_id, created_at DESC, id DESC" + " ) np ON np.property_id = p.id" " WHERE p.portfolio_id = :portfolio_id" - " AND pl.scenario_id = :scenario_id" - " AND pl.is_default = TRUE" - " AND p.id = ANY(:property_ids)" " ORDER BY p.id" ) -# The selected measures on each Property's default Plan: the recommendations -# kept in the Optimised Package (``default`` and not ``already_installed``), -# with the installed material's battery flag for the solar pivot (ADR-0065). -# ``default`` is a reserved word, hence the quoting. +# The selected measures on each Property's NEWEST Plan under the Scenario: the +# recommendations kept in the Optimised Package (``default`` and not +# ``already_installed``), with the installed material's battery flag for the +# solar pivot (ADR-0065). ``default`` is a reserved word, hence the quoting. _MEASURES_SQL = text( "SELECT r.property_id, r.measure_type, r.estimated_cost, r.sap_points," " r.co2_equivalent_savings, r.kwh_savings, r.energy_cost_savings," " COALESCE(m.includes_battery, FALSE) AS includes_battery" " FROM recommendation r" - " JOIN plan pl ON pl.id = r.plan_id" + " JOIN (" + " SELECT DISTINCT ON (property_id) id" + " FROM plan" + " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" + " AND property_id = ANY(:property_ids)" + " ORDER BY property_id, created_at DESC, id DESC" + " ) np ON np.id = r.plan_id" " LEFT JOIN material m ON m.id = r.material_id" - " WHERE pl.portfolio_id = :portfolio_id" - " AND pl.scenario_id = :scenario_id" - " AND pl.is_default = TRUE" - " AND pl.property_id = ANY(:property_ids)" - ' AND r."default" = TRUE' + ' WHERE r."default" = TRUE' " AND r.already_installed = FALSE" ) -# The Effective-EPC descriptive picture, read from the persisted lodged EPC slot -# (ADR-0065). Header facts, descriptive energy elements, and current performance -# are read separately and composed per Property. +# The Effective-EPC descriptive picture (ADR-0065). Header facts and current +# performance are matched by UPRN (the latest epc_property for the source), +# because the PasHub re-fetch lands records with property_id = NULL — a +# property_id join would silently fall back to a stale gov-EPC cert (wrong +# lodgement dates / property_type) or nothing. The descriptive prose elements +# have no PasHub equivalent (a survey lodges structured fields, not gov-EPC +# prose), so they stay on the property_id join as the best-available gov-EPC +# fallback. All three are read separately and composed per Property. _EPC_HEADER_SQL = text( - "SELECT property_id, property_type, total_floor_area_m2, registration_date," - " habitable_rooms_count" - " FROM epc_property" - " WHERE property_id = ANY(:property_ids) AND source = :source" + "WITH latest AS (" + " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" + " FROM property p JOIN epc_property e ON e.uprn = p.uprn" + " WHERE p.id = ANY(:property_ids) AND e.source = :source" + " ORDER BY p.id, e.id DESC)" + " SELECT l.property_id, e.property_type, e.total_floor_area_m2," + " COALESCE(e.registration_date, e.completion_date, e.inspection_date)," + " e.habitable_rooms_count" + " FROM latest l JOIN epc_property e ON e.id = l.epc_id" ) _EPC_ELEMENTS_SQL = text( "SELECT ep.property_id, e.element_type, e.description" @@ -63,11 +82,15 @@ _EPC_ELEMENTS_SQL = text( " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" ) _EPC_PERF_SQL = text( - "SELECT ep.property_id, perf.energy_rating_current," + "WITH latest AS (" + " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" + " FROM property p JOIN epc_property e ON e.uprn = p.uprn" + " WHERE p.id = ANY(:property_ids) AND e.source = :source" + " ORDER BY p.id, e.id DESC)" + " SELECT l.property_id, perf.energy_rating_current," " perf.current_energy_efficiency_band" - " FROM epc_property_energy_performance perf" - " JOIN epc_property ep ON ep.id = perf.epc_property_id" - " WHERE ep.property_id = ANY(:property_ids) AND ep.source = :source" + " FROM latest l" + " JOIN epc_property_energy_performance perf ON perf.epc_property_id = l.epc_id" ) # A landlord property_type override wins over any EPC-derived value (ADR-0065); diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index b5940eb33..8d6f77bdd 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -34,6 +34,7 @@ def _add_epc( *, property_id: int, epc_id: int, + uprn: int = 100001, source: str = "lodged", property_type: Optional[str] = None, total_floor_area_m2: float = 90.0, @@ -50,6 +51,7 @@ def _add_epc( EpcPropertyModel( id=epc_id, property_id=property_id, + uprn=uprn, source=source, property_type=property_type, total_floor_area_m2=total_floor_area_m2, @@ -147,12 +149,13 @@ def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( assert row.cost_of_works == 4200.0 -def test_excludes_requested_properties_without_a_default_plan_under_the_scenario( +def test_excludes_requested_properties_without_a_plan_under_the_scenario( db_engine: Engine, ) -> None: - # arrange — four requested Properties: only #1 has a default Plan under - # scenario 5. #2 has a non-default Plan there, #3 a default Plan under a - # different scenario, #4 no Plan at all. + # arrange — four requested Properties: #1 and #2 both have a Plan under + # scenario 5 (#2's is non-default — is_default no longer gates, since it is + # one-per-property not per-scenario), #3 a Plan under a different scenario, + # #4 no Plan at all. with Session(db_engine) as session: for pid in (1, 2, 3, 4): session.add( @@ -175,8 +178,43 @@ def test_excludes_requested_properties_without_a_default_plan_under_the_scenario portfolio_id=100, scenario_id=5, property_ids=[1, 2, 3, 4] ) - # assert — only #1 (a default Plan under scenario 5) survives (model A). - assert [r.property_id for r in rows] == [1] + # assert — #1 and #2 (both have a Plan under scenario 5) survive; #3 (wrong + # scenario) and #4 (no Plan) drop out. + assert [r.property_id for r in rows] == [1, 2] + + +def test_selects_the_newest_plan_per_property_under_the_scenario( + db_engine: Engine, +) -> None: + # arrange — a Property with an older and a newer Plan under scenario 5; the + # newest Plan wins regardless of is_default (a re-model appends a new Plan). + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel( + id=1, portfolio_id=100, property_id=1, scenario_id=5, + is_default=True, post_sap_points=60.0, + ) + ) + session.add( + PlanModel( + id=2, portfolio_id=100, property_id=1, scenario_id=5, + is_default=False, post_sap_points=72.0, + ) + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the newer Plan (id=2, non-default) wins over the older default. + assert len(rows) == 1 + assert rows[0].post_sap_points == 72.0 def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( @@ -186,7 +224,9 @@ def test_maps_the_default_plans_selected_measures_with_battery_and_exclusions( # material carries a battery), plus a non-default and an already-installed # recommendation that must be excluded. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel( id=10, portfolio_id=100, property_id=1, scenario_id=5, is_default=True @@ -273,7 +313,9 @@ def test_effective_epc_descriptive_fields_come_from_the_lodged_epc( # arrange — a Property with a default Plan under scenario 5 and a lodged EPC # carrying descriptive energy elements and current performance. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) ) @@ -323,7 +365,9 @@ def test_property_type_override_beats_the_lodged_epc(db_engine: Engine) -> None: # arrange — the lodged EPC says Flat, but a landlord override says House # (Effective-EPC precedence: override → lodged, ADR-0065). with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) ) @@ -355,7 +399,9 @@ def test_falls_back_to_the_predicted_epc_when_there_is_no_lodged_epc( ) -> None: # arrange — the Property has only a predicted EPC slot, no lodged cert. with Session(db_engine) as session: - session.add(PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1")) + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) session.add( PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) ) From 35a0c6e972e094dc004d0984ab27f8a0150ab543 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 18 Jul 2026 19:38:12 +0000 Subject: [PATCH 43/46] Source export fabric from structured SAP fields; fix stale-cert selection The "latest EPC per UPRN" selection ordered by e.id DESC, but row-id does not track recency: a gov-EPC cert re-ingested after a PasHub survey lands with a higher e.id, so the header/perf reads silently picked the stale gov cert (wrong TFA, lodgement, property_type, current band) on 3 of portfolio 838. Order by the effective lodgement date (registration -> completion -> inspection) DESC NULLS LAST, e.id DESC tiebreak. Compose the descriptive fabric from the structured SAP fields a re-survey lodges (no gov-EPC prose): walls/roof/floor from the main epc_building_part, heating and hot water from epc_main_heating_detail + the water-heating codes, decoded with the SAP engine's own code space (fabric_description.py). Structured fabric wins where present and falls back to the gov-EPC prose otherwise; windows and lighting stay on prose (their per-element codes do not reconstruct the dwelling phrase). Add built_form (epc_property, code-decoded) and property_age_band (building part construction_age_band -> RdSAP date range) columns. Co-Authored-By: Claude Opus 4.8 (1M context) --- domain/scenario_export/fabric_description.py | 284 ++++++++++++++++++ domain/scenario_export/scenario_sheet.py | 6 + .../scenario_export_postgres_repository.py | 96 +++++- .../test_fabric_description.py | 174 +++++++++++ ...est_scenario_export_postgres_repository.py | 238 ++++++++++++++- 5 files changed, 790 insertions(+), 8 deletions(-) create mode 100644 domain/scenario_export/fabric_description.py create mode 100644 tests/domain/scenario_export/test_fabric_description.py diff --git a/domain/scenario_export/fabric_description.py b/domain/scenario_export/fabric_description.py new file mode 100644 index 000000000..54529f212 --- /dev/null +++ b/domain/scenario_export/fabric_description.py @@ -0,0 +1,284 @@ +"""Human-readable fabric descriptions from an EPC's structured SAP fields +(ADR-0065). + +The PasHub re-survey lodges *structured* fabric — SAP building parts, a main +heating detail, coded fuel/water-heating — but no gov-EPC descriptive prose, so +the export's descriptive columns would be blank for a re-surveyed Property. This +module composes the client-sheet text for walls / roof / floor (from the main +``epc_building_part``) and heating / hot water (from ``epc_main_heating_detail`` +and the ``epc_property`` water-heating fields), decoding the SAP integer codes +with the same code space the SAP engine uses. Windows and lighting are left on +the gov-EPC prose fallback (their per-element codes do not cleanly reconstruct +the dwelling-level phrase). + +Pure logic: the repository reads the structured rows and calls these; each +renderer returns ``None`` when it has nothing to say, so the caller can fall +back to the gov-EPC prose. +""" + +from __future__ import annotations + +from typing import Any, Optional + +# --- SAP integer-code → display-text maps --------------------------------- +# wall_construction (SAP10 codes, domain/sap10_ml/rdsap_uvalues.py:127-142). +_WALL_CONSTRUCTION: dict[int, str] = { + 1: "Granite or whinstone", + 2: "Sandstone", + 3: "Solid brick", + 4: "Cavity", + 5: "Timber frame", + 6: "System build", + 7: "Cob", + 8: "Park home", + 9: "Curtain wall", + 10: "Unknown", + 11: "Cavity (filled party wall)", +} +# wall_insulation_type (RdSAP schema codes, rdsap_uvalues.py:145-158). +_WALL_INSULATION: dict[int, str] = { + 1: "external insulation", + 2: "filled cavity", + 3: "internal insulation", + 4: "as built", + 5: "no insulation", + 6: "filled cavity + external insulation", + 7: "filled cavity + internal insulation", +} +# heat_emitter_type (SAP10.2, mapper.py _ELMHURST_HEAT_EMITTER_TO_SAP10). +_HEAT_EMITTER: dict[int, str] = { + 1: "radiators", + 2: "underfloor heating (in screed)", + 3: "underfloor heating (timber floor)", + 4: "warm air", + 5: "fan coils", +} +# main_fuel_type / water_heating_fuel (RdSAP "(not community)" family; invert of +# main_fuel_overlay._FUEL_CODES). +_FUEL: dict[int, str] = { + 3: "bottled LPG", + 6: "wood logs", + 10: "dual fuel (mineral and wood)", + 15: "smokeless coal", + 17: "LPG (special condition)", + 20: "mains gas (community)", + 25: "electricity (community)", + 26: "mains gas", + 27: "LPG (bulk)", + 28: "oil", + 29: "electricity", + 31: "biomass (community)", + 33: "house coal", +} +# main_heating_control (SAP10.2 Table 4e Group 1; comments in +# heating_recommendation._CONTROL_FEATURES_BY_CODE). +_HEATING_CONTROL: dict[int, str] = { + 2101: "no time or thermostatic control", + 2102: "programmer, no room thermostat", + 2103: "room thermostat only", + 2104: "programmer and room thermostat", + 2105: "programmer and at least two room thermostats", + 2106: "programmer, room thermostat and TRVs", + 2107: "programmer, TRVs and bypass", + 2108: "programmer, TRVs and flow switch", + 2109: "programmer, TRVs and boiler energy manager", + 2111: "TRVs and bypass", + 2113: "room thermostat and TRVs", +} +# water_heating_code (SAP Table 4a; water_heating_overlay docstring). +_WATER_HEATING_SYSTEM: dict[int, str] = { + 901: "from main system", + 903: "electric immersion", + 911: "gas boiler / circulator", +} +# RdSAP England-&-Wales construction age-band letters → the date range the EPC +# prints (domain/epc/property_overrides/construction_age_band.py). The lodged +# `epc_building_part.construction_age_band` carries the bare letter. +_AGE_BAND: dict[str, str] = { + "A": "before 1900", + "B": "1900-1929", + "C": "1930-1949", + "D": "1950-1966", + "E": "1967-1975", + "F": "1976-1982", + "G": "1983-1990", + "H": "1991-1995", + "I": "1996-2002", + "J": "2003-2006", + "K": "2007-2011", + "L": "2012-2022", + "M": "2023 onwards", +} + + +def _as_int(value: Any) -> Optional[int]: + """A SAP code as ``int``. Accepts the raw int, a numeric string, or ``None``; + a non-numeric label (e.g. an insulation ``"As built"`` string in a numeric + slot) yields ``None``.""" + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + try: + return int(str(value).strip()) + except ValueError: + return None + + +def _clean_text(value: Any) -> Optional[str]: + """A lodged free-text field trimmed to a non-empty string, else ``None``. + A bare integer (a code lodged in a text slot) is not display text.""" + if value is None or isinstance(value, (int, float, bool)): + return None + text = str(value).strip() + return text or None + + +# built_form codes (epc_codes.csv; override_code_mapping._BUILT_FORM_CODES). +# The lodged value is usually the display text already; a minority of records +# carry the bare integer code instead. +_BUILT_FORM: dict[int, str] = { + 1: "Detached", + 2: "Semi-detached", + 3: "End-terrace", + 4: "Mid-terrace", + 5: "Enclosed end-terrace", + 6: "Enclosed mid-terrace", +} + + +def built_form_description(built_form: Any) -> Optional[str]: + """The dwelling's built form. Decodes the integer code where a record + lodges one, otherwise passes the lodged display text through unchanged.""" + code = _as_int(built_form) + if code is not None: + return _BUILT_FORM.get(code) + return _clean_text(built_form) + + +def age_band_description(band: Any) -> Optional[str]: + """The construction age band as its date range, e.g. ``"1900-1929 (band B)"``. + The lodged value is a bare RdSAP letter; an unrecognised or ``"Unknown"`` + band renders nothing.""" + letter = _clean_text(band) + if letter is None: + return None + date_range = _AGE_BAND.get(letter.strip().upper()) + return f"{date_range} (band {letter.strip().upper()})" if date_range else None + + +def wall_description(construction: Any, insulation: Any) -> Optional[str]: + """A wall part's text, e.g. ``"Cavity wall, filled cavity"``. The + construction names the wall; the insulation qualifies it when known.""" + cons = _WALL_CONSTRUCTION.get(_as_int(construction) or 0) + if cons is None: + return None + ins = _WALL_INSULATION.get(_as_int(insulation) or 0) + return f"{cons} wall, {ins}" if ins else f"{cons} wall" + + +def _insulation_thickness_mm(thickness: Any) -> Optional[int]: + """A roof-insulation thickness in mm from a numeric code or a ``"225mm"`` + string; a non-numeric label (``"As built"``, ``"ND"``) or ``None`` yields + ``None``. Zero is a real value (no insulation) and is preserved.""" + as_int = _as_int(thickness) + if as_int is not None: + return as_int + text = _clean_text(thickness) + if text is None: + return None + digits = "".join(ch for ch in text if ch.isdigit()) + return int(digits) if digits else None + + +def roof_description( + construction_type: Any, insulation_location: Any, insulation_thickness: Any +) -> Optional[str]: + """A roof part's text, e.g. ``"Pitched roof (Slates or tiles), Access to + loft; 270mm insulation at joists"``. The construction is already lodged as + prose; the insulation clause is appended when a thickness or a named + location is present.""" + cons = _clean_text(construction_type) + if cons is None: + return None + thickness_mm = _insulation_thickness_mm(insulation_thickness) + location = _clean_text(insulation_location) + if location is not None and location.lower() in {"unknown", "none"}: + location = None + if thickness_mm == 0: + return f"{cons}; no insulation" + if thickness_mm is None and location is None: + return cons + clause = "insulation" if thickness_mm is None else f"{thickness_mm}mm insulation" + if location is not None: + clause = f"{clause} at {location.lower()}" + return f"{cons}; {clause}" + + +def floor_description( + floor_type: Any, construction_type: Any, insulation: Any +) -> Optional[str]: + """A floor part's text, e.g. ``"Ground floor, suspended timber, insulation + as built"``. Composed from the lodged floor type, construction, and + insulation status (all lodged as prose).""" + parts: list[str] = [] + location = _clean_text(floor_type) + if location is not None: + parts.append(location[0].upper() + location[1:].lower()) + construction = _clean_text(construction_type) + if construction is not None: + parts.append(construction.lower()) + ins = _clean_text(insulation) + if ins is not None: + parts.append(f"insulation {ins.lower()}") + return ", ".join(parts) or None + + +def heating_description( + fuel: Any, emitter: Any, control: Any, condensing: Any +) -> Optional[str]: + """The main heating system's text from its SAP codes, e.g. ``"Mains gas, + radiators (condensing); programmer, room thermostat and TRVs"``. Reports + only what the codes state — the appliance archetype is not lodged on a + re-survey, so no boiler/heat-pump type is asserted.""" + fuel_text = _FUEL.get(_as_int(fuel) or 0) + emitter_text = _HEAT_EMITTER.get(_as_int(emitter) or 0) + lead_parts = [p for p in (fuel_text, emitter_text) if p] + if not lead_parts: + return None + lead = ", ".join(p[0].upper() + p[1:] if i == 0 else p for i, p in enumerate(lead_parts)) + if condensing is True: + lead = f"{lead} (condensing)" + control_text = _HEATING_CONTROL.get(_as_int(control) or 0) + return f"{lead}; {control_text}" if control_text else lead + + +def hot_water_description( + water_heating_code: Any, + water_heating_fuel: Any, + main_fuel: Any, + has_cylinder: Any, + solar: Any, +) -> Optional[str]: + """The hot-water system's text from its SAP codes, e.g. ``"From main + system, mains gas"``. Where the water-heating code is not lodged on the + re-survey, it falls back to the SAP default (a dwelling with no cylinder + draws hot water from the main system) using the main heating fuel.""" + system = _WATER_HEATING_SYSTEM.get(_as_int(water_heating_code) or 0) + fuel = _FUEL.get(_as_int(water_heating_fuel) or 0) or _FUEL.get(_as_int(main_fuel) or 0) + if system is None: + # No lodged water-heating code: a dwelling without a cylinder inherits + # hot water from the main system (SAP Table 4a code 901). + if has_cylinder is True: + system = "hot water cylinder" + elif has_cylinder is False: + system = "from main system" + else: + return None + text = f"{system}, {fuel}" if fuel else system + text = text[0].upper() + text[1:] + if solar is True: + text = f"{text}; with solar water heating" + return text diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index 018b89e11..ae3860d23 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -59,6 +59,8 @@ _ID_COLUMNS: tuple[str, ...] = ( "address", "postcode", "property_type", + "built_form", + "property_age_band", "walls", "roof", "floor", @@ -147,6 +149,8 @@ class PropertyScenarioData: address: Optional[str] = None postcode: Optional[str] = None property_type: Optional[str] = None + built_form: Optional[str] = None + property_age_band: Optional[str] = None walls: Optional[str] = None roof: Optional[str] = None floor: Optional[str] = None @@ -196,6 +200,8 @@ def _shape_row(prop: PropertyScenarioData) -> dict[str, Any]: "address": _blank(prop.address), "postcode": _blank(prop.postcode), "property_type": _blank(prop.property_type), + "built_form": _blank(prop.built_form), + "property_age_band": _blank(prop.property_age_band), "walls": _blank(prop.walls), "roof": _blank(prop.roof), "floor": _blank(prop.floor), diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index a2674b20f..50e21dd66 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -6,6 +6,15 @@ from typing import Any, Optional, Sequence from sqlalchemy import text from sqlmodel import Session +from domain.scenario_export.fabric_description import ( + age_band_description, + built_form_description, + floor_description, + heating_description, + hot_water_description, + roof_description, + wall_description, +) from domain.scenario_export.scenario_sheet import ExportMeasure, PropertyScenarioData from repositories.scenario_export.scenario_export_repository import ( ScenarioExportRepository, @@ -60,19 +69,26 @@ _MEASURES_SQL = text( # performance are matched by UPRN (the latest epc_property for the source), # because the PasHub re-fetch lands records with property_id = NULL — a # property_id join would silently fall back to a stale gov-EPC cert (wrong -# lodgement dates / property_type) or nothing. The descriptive prose elements -# have no PasHub equivalent (a survey lodges structured fields, not gov-EPC -# prose), so they stay on the property_id join as the best-available gov-EPC -# fallback. All three are read separately and composed per Property. +# lodgement dates / property_type) or nothing. "Latest" is ordered by the cert's +# effective lodgement date (registration → completion → inspection) DESC, NOT by +# `e.id`: row-id order does not track recency — a gov-EPC cert re-ingested after +# the PasHub survey lands with a HIGHER `e.id`, so an `e.id` sort silently picks +# the stale gov cert over the fresh 2026 survey (observed on 3 of portfolio 838). +# `e.id DESC` remains the tiebreak within one date (duplicate survey loads carry +# identical facts). The descriptive prose elements have no PasHub equivalent (a +# survey lodges structured fields, not gov-EPC prose), so they stay on the +# property_id join as the best-available gov-EPC fallback. All three are read +# separately and composed per Property. _EPC_HEADER_SQL = text( "WITH latest AS (" " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" " FROM property p JOIN epc_property e ON e.uprn = p.uprn" " WHERE p.id = ANY(:property_ids) AND e.source = :source" - " ORDER BY p.id, e.id DESC)" + " ORDER BY p.id, COALESCE(e.registration_date, e.completion_date," + " e.inspection_date) DESC NULLS LAST, e.id DESC)" " SELECT l.property_id, e.property_type, e.total_floor_area_m2," " COALESCE(e.registration_date, e.completion_date, e.inspection_date)," - " e.habitable_rooms_count" + " e.habitable_rooms_count, e.built_form" " FROM latest l JOIN epc_property e ON e.id = l.epc_id" ) _EPC_ELEMENTS_SQL = text( @@ -86,13 +102,57 @@ _EPC_PERF_SQL = text( " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" " FROM property p JOIN epc_property e ON e.uprn = p.uprn" " WHERE p.id = ANY(:property_ids) AND e.source = :source" - " ORDER BY p.id, e.id DESC)" + " ORDER BY p.id, COALESCE(e.registration_date, e.completion_date," + " e.inspection_date) DESC NULLS LAST, e.id DESC)" " SELECT l.property_id, perf.energy_rating_current," " perf.current_energy_efficiency_band" " FROM latest l" " JOIN epc_property_energy_performance perf ON perf.epc_property_id = l.epc_id" ) +# The structured SAP fabric on the same canonical (latest-by-date, UPRN-matched) +# cert the header reads: the MAIN building part (walls / roof / floor), the main +# heating detail (fuel / emitter / control / condensing) and the water-heating +# fields. A PasHub re-survey lodges these coded facts but no gov-EPC prose, so +# they supply walls/roof/floor/heating/hot_water where the prose fallback is +# blank. The main building part is chosen identifier='main' first, then the +# lowest part number / id (matching the SAP engine's `parts[0]` primary part). +# Windows and lighting are not read here (their per-element codes do not cleanly +# reconstruct the dwelling phrase); they stay on the gov-EPC prose fallback. +_EPC_STRUCTURED_FABRIC_SQL = text( + "WITH latest AS (" + " SELECT DISTINCT ON (p.id) p.id AS property_id, e.id AS epc_id" + " FROM property p JOIN epc_property e ON e.uprn = p.uprn" + " WHERE p.id = ANY(:property_ids) AND e.source = :source" + " ORDER BY p.id, COALESCE(e.registration_date, e.completion_date," + " e.inspection_date) DESC NULLS LAST, e.id DESC)," + " main_part AS (" + " SELECT DISTINCT ON (l.property_id) l.property_id, bp.construction_age_band," + " bp.wall_construction, bp.wall_insulation_type, bp.roof_construction_type," + " bp.roof_insulation_location, bp.roof_insulation_thickness, bp.floor_type," + " bp.floor_construction_type, bp.floor_insulation_type_str" + " FROM latest l JOIN epc_building_part bp ON bp.epc_property_id = l.epc_id" + " ORDER BY l.property_id," + " (CASE WHEN lower(bp.identifier) = 'main' THEN 0 ELSE 1 END)," + " bp.building_part_number NULLS FIRST, bp.id)," + " main_heat AS (" + " SELECT DISTINCT ON (h.epc_property_id) h.epc_property_id, h.main_fuel_type," + " h.heat_emitter_type, h.main_heating_control, h.condensing" + " FROM epc_main_heating_detail h" + " ORDER BY h.epc_property_id, h.main_heating_number NULLS FIRST, h.id)" + " SELECT l.property_id, mp.wall_construction, mp.wall_insulation_type," + " mp.roof_construction_type, mp.roof_insulation_location," + " mp.roof_insulation_thickness, mp.floor_type, mp.floor_construction_type," + " mp.floor_insulation_type_str, mh.main_fuel_type, mh.heat_emitter_type," + " mh.main_heating_control, mh.condensing, e.heating_water_heating_code," + " e.heating_water_heating_fuel, e.has_hot_water_cylinder," + " e.solar_water_heating, mp.construction_age_band" + " FROM latest l" + " JOIN epc_property e ON e.id = l.epc_id" + " LEFT JOIN main_part mp ON mp.property_id = l.property_id" + " LEFT JOIN main_heat mh ON mh.epc_property_id = l.epc_id" +) + # A landlord property_type override wins over any EPC-derived value (ADR-0065); # the descriptive prose fields have no override prose, so only property_type is # resolved this way (the override enum carries coded fabric facts, not prose). @@ -183,6 +243,8 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): energy_consumption_savings=_float(row[11]), valuation_increase=_float(row[12]), property_type=facts.get("property_type"), + built_form=facts.get("built_form"), + property_age_band=facts.get("property_age_band"), walls=facts.get("walls"), roof=facts.get("roof"), floor=facts.get("floor"), @@ -261,6 +323,7 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): "lodgement_date": registration_date, "is_expired": _is_expired(registration_date), "number_of_rooms": row[4], + "built_form": built_form_description(row[5]), } ) @@ -287,4 +350,23 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): } ) + # Structured SAP fabric (walls/roof/floor/heating/hot_water) composed + # from the canonical cert's coded fields; overrides the gov-EPC prose + # where present (a re-survey lodges these but no prose), and leaves the + # prose in place where a field renders to nothing. + for row in connection.execute(_EPC_STRUCTURED_FABRIC_SQL, source_params).all(): + rendered = { + "property_age_band": age_band_description(row[17]), + "walls": wall_description(row[1], row[2]), + "roof": roof_description(row[3], row[4], row[5]), + "floor": floor_description(row[6], row[7], row[8]), + "heating": heating_description(row[9], row[10], row[11], row[12]), + "hot_water": hot_water_description( + row[13], row[14], row[9], row[15], row[16] + ), + } + facts.setdefault(row[0], {}).update( + {field: value for field, value in rendered.items() if value is not None} + ) + return facts diff --git a/tests/domain/scenario_export/test_fabric_description.py b/tests/domain/scenario_export/test_fabric_description.py new file mode 100644 index 000000000..6d0397c1f --- /dev/null +++ b/tests/domain/scenario_export/test_fabric_description.py @@ -0,0 +1,174 @@ +"""Behaviour of the structured-fabric renderers (ADR-0065): composing client +sheet text from an EPC's SAP integer codes, decoding with the SAP engine's own +code space and normalising the ragged shapes a re-survey lodges.""" + +from __future__ import annotations + +from domain.scenario_export.fabric_description import ( + age_band_description, + built_form_description, + floor_description, + heating_description, + hot_water_description, + roof_description, + wall_description, +) + + +class TestBuiltFormDescription: + def test_lodged_display_text_passes_through(self) -> None: + assert built_form_description("Mid-terrace") == "Mid-terrace" + assert built_form_description("Semi-Detached") == "Semi-Detached" + + def test_integer_code_is_decoded(self) -> None: + assert built_form_description(4) == "Mid-terrace" + assert built_form_description("1") == "Detached" + + def test_absent_or_unknown_code_renders_nothing(self) -> None: + assert built_form_description(None) is None + assert built_form_description(99) is None + + +class TestAgeBandDescription: + def test_rdsap_letter_decodes_to_date_range_and_band(self) -> None: + assert age_band_description("B") == "1900-1929 (band B)" + assert age_band_description("A") == "before 1900 (band A)" + assert age_band_description("M") == "2023 onwards (band M)" + + def test_lowercase_and_whitespace_are_tolerated(self) -> None: + assert age_band_description(" k ") == "2007-2011 (band K)" + + def test_unknown_or_absent_band_renders_nothing(self) -> None: + assert age_band_description("Unknown") is None + assert age_band_description(None) is None + assert age_band_description("Z") is None + + +class TestWallDescription: + def test_construction_and_insulation_codes_decode_to_text(self) -> None: + assert wall_description(4, 2) == "Cavity wall, filled cavity" + assert wall_description(3, 4) == "Solid brick wall, as built" + assert wall_description(5, 1) == "Timber frame wall, external insulation" + + def test_construction_only_when_insulation_unknown(self) -> None: + assert wall_description(4, None) == "Cavity wall" + assert wall_description(4, 99) == "Cavity wall" + + def test_numeric_string_codes_are_accepted(self) -> None: + # JSONB can round-trip a code as a string; it still decodes. + assert wall_description("4", "2") == "Cavity wall, filled cavity" + + def test_unknown_construction_renders_nothing(self) -> None: + assert wall_description(None, 2) is None + assert wall_description(99, 2) is None + + +class TestRoofDescription: + def test_construction_prose_with_thickness_and_location(self) -> None: + assert ( + roof_description("Pitched roof, Access to loft", "Joists", 270) + == "Pitched roof, Access to loft; 270mm insulation at joists" + ) + + def test_thickness_string_with_unit_is_normalised(self) -> None: + # A minority of records lodge the thickness as "225mm" rather than 225. + assert ( + roof_description("Pitched", "Joists", "225mm") + == "Pitched; 225mm insulation at joists" + ) + + def test_non_numeric_thickness_and_coded_location_are_dropped(self) -> None: + # thickness "As built" is not a measurement; a bare int location is a + # code, not a place — with neither, only the construction survives. + assert roof_description("Pitched", 2, "As built") == "Pitched" + + def test_named_location_without_a_thickness_still_notes_insulation(self) -> None: + assert ( + roof_description("Pitched", "Joists", "As built") + == "Pitched; insulation at joists" + ) + + def test_zero_thickness_reads_as_no_insulation(self) -> None: + assert roof_description("Flat roof", "Joists", 0) == "Flat roof; no insulation" + + def test_unknown_location_is_omitted(self) -> None: + assert roof_description("Pitched", "Unknown", 100) == "Pitched; 100mm insulation" + + def test_construction_only_when_no_insulation_detail(self) -> None: + assert roof_description("Pitched", None, None) == "Pitched" + + def test_no_construction_renders_nothing(self) -> None: + assert roof_description(None, "Joists", 270) is None + + +class TestFloorDescription: + def test_composes_type_construction_and_insulation(self) -> None: + assert ( + floor_description("Ground floor", "Suspended, timber", "As Built") + == "Ground floor, suspended, timber, insulation as built" + ) + + def test_normalises_casing_of_the_floor_type(self) -> None: + assert floor_description("Ground Floor", "Solid", None) == "Ground floor, solid" + + def test_retrofitted_insulation_is_carried(self) -> None: + assert ( + floor_description("Ground floor", "Solid", "Retro-fitted") + == "Ground floor, solid, insulation retro-fitted" + ) + + def test_empty_when_nothing_lodged(self) -> None: + assert floor_description(None, None, None) is None + + +class TestHeatingDescription: + def test_fuel_emitter_control_and_condensing(self) -> None: + assert ( + heating_description(26, 1, 2106, True) + == "Mains gas, radiators (condensing); programmer, room thermostat and TRVs" + ) + + def test_condensing_false_or_absent_omits_the_flag(self) -> None: + assert heating_description(26, 1, 2104, None) == ( + "Mains gas, radiators; programmer and room thermostat" + ) + assert heating_description(26, 1, None, False) == "Mains gas, radiators" + + def test_unknown_control_is_dropped(self) -> None: + assert heating_description(26, 1, 9999, True) == "Mains gas, radiators (condensing)" + + def test_empty_when_no_fuel_or_emitter(self) -> None: + assert heating_description(None, None, 2106, True) is None + + +class TestHotWaterDescription: + def test_lodged_code_and_fuel_decode(self) -> None: + assert ( + hot_water_description(901, 26, 26, False, False) == "From main system, mains gas" + ) + assert ( + hot_water_description(903, 29, 26, True, False) + == "Electric immersion, electricity" + ) + + def test_falls_back_to_main_system_when_no_code_and_no_cylinder(self) -> None: + # SAP Table 4a: a dwelling with no cylinder draws hot water from the main + # system; the main heating fuel names it. + assert ( + hot_water_description(None, None, 26, False, False) + == "From main system, mains gas" + ) + + def test_cylinder_without_a_code(self) -> None: + assert ( + hot_water_description(None, None, 29, True, False) + == "Hot water cylinder, electricity" + ) + + def test_solar_water_heating_is_appended(self) -> None: + assert hot_water_description(901, 26, 26, False, True) == ( + "From main system, mains gas; with solar water heating" + ) + + def test_empty_when_nothing_lodged(self) -> None: + assert hot_water_description(None, None, None, None, False) is None diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index 8d6f77bdd..0210502ad 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -4,14 +4,16 @@ read-model, from persisted data only (no live EPC calls).""" from __future__ import annotations -from typing import Optional +from typing import Optional, Union from sqlalchemy import Engine from sqlmodel import Session from datatypes.epc.domain.epc import Epc from infrastructure.postgres.epc_property_table import ( # registers epc tables + EpcBuildingPartModel, EpcEnergyElementModel, + EpcMainHeatingDetailModel, EpcPropertyEnergyPerformanceModel, EpcPropertyModel, ) @@ -37,6 +39,7 @@ def _add_epc( uprn: int = 100001, source: str = "lodged", property_type: Optional[str] = None, + built_form: Optional[str] = None, total_floor_area_m2: float = 90.0, registration_date: str = "2005-01-01", habitable_rooms_count: int = 4, @@ -54,6 +57,7 @@ def _add_epc( uprn=uprn, source=source, property_type=property_type, + built_form=built_form, total_floor_area_m2=total_floor_area_m2, registration_date=registration_date, habitable_rooms_count=habitable_rooms_count, @@ -103,6 +107,66 @@ def _add_epc( ) +def _add_building_part( + session: Session, + *, + epc_id: int, + identifier: str = "main", + building_part_number: Optional[int] = None, + wall_construction: int = 4, + wall_insulation_type: int = 2, + roof_construction_type: Optional[str] = "Pitched roof, Access to loft", + roof_insulation_location: Optional[str] = "Joists", + roof_insulation_thickness: Optional[Union[str, int]] = 270, + floor_type: Optional[str] = "Ground floor", + floor_construction_type: Optional[str] = "Solid", + floor_insulation_type_str: Optional[str] = "As Built", +) -> None: + """Seed one SAP building part on an EPC slot (the source of the structured + walls / roof / floor text).""" + session.add( + EpcBuildingPartModel( + epc_property_id=epc_id, + identifier=identifier, + construction_age_band="D", + wall_construction=wall_construction, + wall_insulation_type=wall_insulation_type, + wall_thickness_measured=False, + building_part_number=building_part_number, + roof_construction_type=roof_construction_type, + roof_insulation_location=roof_insulation_location, + roof_insulation_thickness=roof_insulation_thickness, + floor_type=floor_type, + floor_construction_type=floor_construction_type, + floor_insulation_type_str=floor_insulation_type_str, + ) + ) + + +def _add_main_heating( + session: Session, + *, + epc_id: int, + main_fuel_type: int = 26, + heat_emitter_type: int = 1, + main_heating_control: int = 2106, + condensing: Optional[bool] = True, +) -> None: + """Seed the main heating detail on an EPC slot (the source of the structured + heating text).""" + session.add( + EpcMainHeatingDetailModel( + epc_property_id=epc_id, + has_fghrs=False, + main_fuel_type=main_fuel_type, + heat_emitter_type=heat_emitter_type, + emitter_temperature=1, + main_heating_control=main_heating_control, + condensing=condensing, + ) + ) + + def test_returns_a_row_for_a_property_with_a_default_plan_under_the_scenario( db_engine: Engine, ) -> None: @@ -361,6 +425,178 @@ def test_effective_epc_descriptive_fields_come_from_the_lodged_epc( assert row.is_expired is True +def test_header_picks_the_freshest_cert_by_date_not_by_row_id( + db_engine: Engine, +) -> None: + # arrange — two lodged certs on one UPRN: a fresh 2026 survey lodged with a + # LOWER epc_id, and a stale 2013 gov cert re-ingested afterwards with a + # HIGHER epc_id. An `e.id DESC` sort would wrongly pick the stale gov cert; + # the freshest lodgement date must win (the portfolio-838 TFA regression). + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc( + session, + property_id=1, + epc_id=10, # fresh survey, LOWER id + uprn=100001, + total_floor_area_m2=76.62, + registration_date="2026-05-18", + ) + _add_epc( + session, + property_id=1, + epc_id=20, # stale gov cert re-ingested later, HIGHER id + uprn=100001, + total_floor_area_m2=77.0, + registration_date="2013-10-09", + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the fresh 2026 survey's floor area and lodgement win, not the + # higher-id 2013 gov cert. + assert rows[0].total_floor_area == 76.62 + assert rows[0].lodgement_date == "2026-05-18" + + +def test_structured_fabric_supplies_walls_roof_floor_heating_and_hot_water( + db_engine: Engine, +) -> None: + # arrange — a lodged EPC carrying gov prose for every descriptive field AND + # the structured SAP fabric (building part + heating detail) a re-survey + # lodges. The structured fabric must win for walls/roof/floor/heating/ + # hot_water; windows/lighting have no structured source and keep the prose. + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc( + session, + property_id=1, + epc_id=1, + source="lodged", + built_form="Mid-terrace", + elements={ + "wall": "PROSE walls", + "roof": "PROSE roof", + "floor": "PROSE floor", + "main_heating": "PROSE heating", + "hot_water": "PROSE hot water", + "window": "Fully double glazed", + "lighting": "Low energy lighting in all fixed outlets", + }, + ) + _add_building_part(session, epc_id=1) + _add_main_heating(session, epc_id=1) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — structured fabric replaces the prose for its five fields; windows + # and lighting (no structured source) keep the gov prose. + row = rows[0] + assert row.built_form == "Mid-terrace" + assert row.property_age_band == "1950-1966 (band D)" + assert row.walls == "Cavity wall, filled cavity" + assert row.roof == "Pitched roof, Access to loft; 270mm insulation at joists" + assert row.floor == "Ground floor, solid, insulation as built" + assert row.heating == ( + "Mains gas, radiators (condensing); programmer, room thermostat and TRVs" + ) + assert row.hot_water == "From main system, mains gas" + assert row.windows == "Fully double glazed" + assert row.lighting == "Low energy lighting in all fixed outlets" + + +def test_structured_fabric_reads_the_main_building_part_over_an_extension( + db_engine: Engine, +) -> None: + # arrange — two building parts on the cert: an extension (solid brick) and + # the main dwelling (cavity). The main part must supply the wall text. + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc(session, property_id=1, epc_id=1, source="lodged") + _add_building_part( + session, + epc_id=1, + identifier="extension_1", + building_part_number=2, + wall_construction=3, # solid brick — the extension + ) + _add_building_part( + session, + epc_id=1, + identifier="main", + building_part_number=1, + wall_construction=4, # cavity — the main dwelling + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — the main (cavity) part wins, not the extension (solid brick). + assert rows[0].walls == "Cavity wall, filled cavity" + + +def test_prose_remains_where_the_structured_fabric_is_absent( + db_engine: Engine, +) -> None: + # arrange — a lodged EPC with gov prose but no building part and no heating + # detail (an older gov cert, not a re-survey). + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=100001) + ) + session.add( + PlanModel(portfolio_id=100, property_id=1, scenario_id=5, is_default=True) + ) + _add_epc( + session, + property_id=1, + epc_id=1, + source="lodged", + elements={"wall": "Cavity wall, as built", "roof": "Pitched, 250mm"}, + ) + session.commit() + + # act + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).rows_for( + portfolio_id=100, scenario_id=5, property_ids=[1] + ) + + # assert — with no structured fabric, the gov prose is the fallback. + row = rows[0] + assert row.walls == "Cavity wall, as built" + assert row.roof == "Pitched, 250mm" + + def test_property_type_override_beats_the_lodged_epc(db_engine: Engine) -> None: # arrange — the lodged EPC says Flat, but a landlord override says House # (Effective-EPC precedence: override → lodged, ADR-0065). From f1e190293b851bca3794c96a90751312f06fd842 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 18 Jul 2026 19:51:51 +0000 Subject: [PATCH 44/46] Address review: stream workbook to /tmp, repo the scenario-name SQL Dan's review on #1546: - Stream the workbook to a /tmp temp file and S3Client.upload_file it, instead of render_workbook -> bytes -> put_object. render_workbook now saves straight to a path; the orchestrator renders to a temp file, multipart-uploads it, and always cleans it up. Restores ADR-0065's "never an in-memory BytesIO" decision (the OOM path at the 100k-row cap). - Move the raw `SELECT ... FROM scenario` out of the Lambda handler into ScenarioNamesPostgresRepository, so the handler stays composition-only and all SQL lives in repositories/. - current_sap_points goes through _float, matching its Optional[float] read-model type and the sibling numeric facts. Co-Authored-By: Claude Opus 4.8 (1M context) --- applications/ara_export/handler.py | 28 +++-------------- .../xlsx/scenario_export_workbook.py | 22 ++++++++----- orchestration/scenario_export_orchestrator.py | 17 ++++++++-- .../scenario_export_postgres_repository.py | 4 ++- .../scenario_names_postgres_repository.py | 31 +++++++++++++++++++ .../xlsx/test_scenario_export_workbook.py | 27 +++++++++------- 6 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 repositories/scenario_export/scenario_names_postgres_repository.py diff --git a/applications/ara_export/handler.py b/applications/ara_export/handler.py index 9e78e2a7b..230441683 100644 --- a/applications/ara_export/handler.py +++ b/applications/ara_export/handler.py @@ -19,8 +19,6 @@ import os from typing import Any import boto3 -from sqlalchemy import text -from sqlmodel import Session from applications.ara_export.ara_export_trigger_body import AraExportTriggerBody from domain.tasks.tasks import Source @@ -32,6 +30,9 @@ from orchestration.scenario_export_orchestrator import ScenarioExportOrchestrato from repositories.scenario_export.scenario_export_postgres_repository import ( ScenarioExportPostgresRepository, ) +from repositories.scenario_export.scenario_names_postgres_repository import ( + ScenarioNamesPostgresRepository, +) from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository from utilities.aws_lambda.task_handler import task_handler @@ -42,27 +43,6 @@ logger = logging.getLogger(__name__) _URL_TTL_SECONDS = 3600 -class _ScenarioNames: - """Scenario id → display name, read from the ``scenario`` table (the domain - Scenario aggregate does not carry the name).""" - - def __init__(self, session: Session, scenario_ids: list[int]) -> None: - rows = ( - session.connection() - .execute( - text("SELECT id, name FROM scenario WHERE id = ANY(:ids)"), - {"ids": scenario_ids}, - ) - .all() - ) - self._names: dict[int, str] = { - row[0]: (row[1] or f"scenario_{row[0]}") for row in rows - } - - def name_for(self, scenario_id: int) -> str: - return self._names.get(scenario_id, f"scenario_{scenario_id}") - - def _email_sender() -> SesSmtpEmailSender: return SesSmtpEmailSender( host=os.environ["SES_SMTP_HOST"], @@ -98,7 +78,7 @@ def handler(body: dict[str, Any], context: Any) -> dict[str, Any]: boto_s3: Any = boto3.client("s3") # pyright: ignore[reportUnknownMemberType] orchestrator = ScenarioExportOrchestrator( rows=ScenarioExportPostgresRepository(session), - scenarios=_ScenarioNames(session, scenario_ids), + scenarios=ScenarioNamesPostgresRepository(session), exports=S3Client(boto_s3, os.environ["DOCUMENT_EXPORTS_BUCKET"]), email=_email_sender(), url_ttl_seconds=_URL_TTL_SECONDS, diff --git a/infrastructure/xlsx/scenario_export_workbook.py b/infrastructure/xlsx/scenario_export_workbook.py index 7dbc9bda8..d437ee0ba 100644 --- a/infrastructure/xlsx/scenario_export_workbook.py +++ b/infrastructure/xlsx/scenario_export_workbook.py @@ -9,7 +9,6 @@ dynamic number of scenario sheets is handled natively. from __future__ import annotations import re -from io import BytesIO from typing import Sequence from openpyxl import Workbook @@ -36,10 +35,19 @@ def _safe_sheet_name(name: str, used: set[str]) -> str: return clean -def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: - """Render the scenario sheets into a single branded ``.xlsx`` as bytes. Each - tuple is a Scenario name and its shaped sheet; sheet order follows the input - (the requested ``scenario_ids`` order).""" +def render_workbook( + sheets: Sequence[tuple[str, ExportSheet]], destination_path: str +) -> None: + """Render the scenario sheets into a single branded ``.xlsx`` at + ``destination_path``. Each tuple is a Scenario name and its shaped sheet; + sheet order follows the input (the requested ``scenario_ids`` order). + + Saves straight to disk (``Workbook.save(path)``) so the caller can + ``S3Client.upload_file`` it with multipart — the workbook is never also held + as an in-memory ``bytes`` copy (ADR-0065; the OOM path the 4 GB sizing was + against at the 100k-row cap). The further memory win is openpyxl + ``write_only`` mode; that switch would live at this seam. + """ workbook = Workbook() default = workbook.active if default is not None: @@ -53,6 +61,4 @@ def render_workbook(sheets: Sequence[tuple[str, ExportSheet]]) -> bytes: worksheet.append([row.get(column, "") for column in sheet.columns]) style_sheet(worksheet) - buffer = BytesIO() - workbook.save(buffer) - return buffer.getvalue() + workbook.save(destination_path) diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py index 282751eb4..39d25eccb 100644 --- a/orchestration/scenario_export_orchestrator.py +++ b/orchestration/scenario_export_orchestrator.py @@ -12,6 +12,8 @@ from __future__ import annotations import html import logging +import os +import tempfile from dataclasses import dataclass from typing import Protocol, Sequence @@ -103,9 +105,20 @@ class ScenarioExportOrchestrator: }, ) - data = render_workbook(named_sheets) key = f"{self._key_prefix}/{export_name}.xlsx" - self._exports.put_object(key, data) + # Render to a /tmp file and multipart-upload it, so the workbook is never + # also held in memory as a bytes copy (ADR-0065 — the OOM path at the + # 100k-row cap). The temp file is always cleaned up. + handle, workbook_path = tempfile.mkstemp(suffix=".xlsx") + os.close(handle) + try: + render_workbook(named_sheets, workbook_path) + self._exports.upload_file(workbook_path, key) + finally: + try: + os.remove(workbook_path) + except OSError: + pass url = self._exports.generate_presigned_url(key, self._url_ttl_seconds) subject, body, html_body = self._compose_email( diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 50e21dd66..c17c220d9 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -345,7 +345,9 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): for row in connection.execute(_EPC_PERF_SQL, source_params).all(): facts.setdefault(row[0], {}).update( { - "current_sap_points": row[1], + # _float to match the read-model's Optional[float] typing and + # the sibling numeric facts (the driver hands back an int). + "current_sap_points": _float(row[1]), "current_epc_rating": _str(row[2]), } ) diff --git a/repositories/scenario_export/scenario_names_postgres_repository.py b/repositories/scenario_export/scenario_names_postgres_repository.py new file mode 100644 index 000000000..690b48023 --- /dev/null +++ b/repositories/scenario_export/scenario_names_postgres_repository.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from sqlalchemy import text +from sqlmodel import Session + +# Scenario id → display name. The domain Scenario aggregate does not carry the +# name, so it is read from the ``scenario`` table; kept in the repository layer +# with the export's other reads (ADR-0065), so the Lambda handler stays +# composition-only. +_SCENARIO_NAME_SQL = text("SELECT name FROM scenario WHERE id = :scenario_id") + + +class ScenarioNamesPostgresRepository: + """Reads a Scenario's display name from Postgres, satisfying the + orchestrator's ``ScenarioNames`` port. Looked up lazily and cached (an export + names only its handful of requested Scenarios).""" + + def __init__(self, session: Session) -> None: + self._session = session + self._cache: dict[int, str] = {} + + def name_for(self, scenario_id: int) -> str: + if scenario_id not in self._cache: + row = ( + self._session.connection() + .execute(_SCENARIO_NAME_SQL, {"scenario_id": scenario_id}) + .first() + ) + name = row[0] if row is not None else None + self._cache[scenario_id] = name or f"scenario_{scenario_id}" + return self._cache[scenario_id] diff --git a/tests/infrastructure/xlsx/test_scenario_export_workbook.py b/tests/infrastructure/xlsx/test_scenario_export_workbook.py index 81e8a3cf8..a1f9e86e6 100644 --- a/tests/infrastructure/xlsx/test_scenario_export_workbook.py +++ b/tests/infrastructure/xlsx/test_scenario_export_workbook.py @@ -3,7 +3,7 @@ sheet per Scenario, headers then rows, verified by re-opening the bytes.""" from __future__ import annotations -from io import BytesIO +from pathlib import Path from openpyxl import load_workbook @@ -12,7 +12,9 @@ from infrastructure.xlsx.branding import HEADER_FILL from infrastructure.xlsx.scenario_export_workbook import render_workbook -def test_renders_a_sheet_named_for_the_scenario_with_headers_then_rows() -> None: +def test_renders_a_sheet_named_for_the_scenario_with_headers_then_rows( + tmp_path: Path, +) -> None: # arrange — one scenario's shaped sheet. sheet = ExportSheet( columns=("property_id", "loft_insulation"), @@ -20,18 +22,19 @@ def test_renders_a_sheet_named_for_the_scenario_with_headers_then_rows() -> None ) # act - data = render_workbook([("Fabric first", sheet)]) + path = str(tmp_path / "export.xlsx") + render_workbook([("Fabric first", sheet)], path) # assert — a real workbook with a sheet named for the scenario, the column # contract as the header row, and the data beneath it. - workbook = load_workbook(BytesIO(data)) + workbook = load_workbook(path) assert workbook.sheetnames == ["Fabric first"] worksheet = workbook["Fabric first"] assert [cell.value for cell in worksheet[1]] == ["property_id", "loft_insulation"] assert [cell.value for cell in worksheet[2]] == [1, 1200.0] -def test_sanitises_and_deduplicates_scenario_sheet_names() -> None: +def test_sanitises_and_deduplicates_scenario_sheet_names(tmp_path: Path) -> None: # arrange — a name over Excel's 31-char limit (used twice, so it must be # deduplicated) and one with characters Excel forbids in a sheet title. empty = ExportSheet(columns=("property_id",), rows=()) @@ -39,19 +42,20 @@ def test_sanitises_and_deduplicates_scenario_sheet_names() -> None: illegal_name = "Bad:/\\?*[]name" # act - data = render_workbook( - [(long_name, empty), (long_name, empty), (illegal_name, empty)] + path = str(tmp_path / "export.xlsx") + render_workbook( + [(long_name, empty), (long_name, empty), (illegal_name, empty)], path ) # assert — three distinct, Excel-legal sheet names. - names = load_workbook(BytesIO(data)).sheetnames + names = load_workbook(path).sheetnames assert len(names) == 3 assert len(set(names)) == 3 assert all(len(name) <= 31 for name in names) assert not any(ch in name for name in names for ch in r":/\?*[]") -def test_brands_the_header_row_and_freezes_it() -> None: +def test_brands_the_header_row_and_freezes_it(tmp_path: Path) -> None: # arrange sheet = ExportSheet( columns=("property_id", "loft_insulation"), @@ -59,11 +63,12 @@ def test_brands_the_header_row_and_freezes_it() -> None: ) # act - data = render_workbook([("Fabric first", sheet)]) + path = str(tmp_path / "export.xlsx") + render_workbook([("Fabric first", sheet)], path) # assert — the header is a bold, brand-filled band and the header row is # frozen so it stays visible while scrolling. - worksheet = load_workbook(BytesIO(data))["Fabric first"] + worksheet = load_workbook(path)["Fabric first"] header_cell = worksheet["A1"] assert header_cell.font.bold is True assert header_cell.fill.fgColor.rgb == HEADER_FILL From 72f3e3a72de69a008a776a0e34912f402610901f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Jul 2026 09:21:38 +0000 Subject: [PATCH 45/46] Add default-plan export selection + plan_name column (ADR-0065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second export selection alongside the per-Scenario one, chosen by `plan_selection` on the export request: - "scenario" (default, unchanged): one sheet per Scenario, freshest Plan per (property, scenario); scenario_ids required. - "default" (new): a single "Default Plans" sheet of each home's is_default Plan (one-per-property across all Scenarios, ADR-0012/0017) — the portfolio's current state, one row per home; scenario_ids ignored. Repository gains `default_rows_for(portfolio_id, property_ids)`: same read-model and Effective-EPC join as `rows_for`, with the plan-selection WHERE swapped from `scenario_id = :scenario_id` to `is_default = TRUE`. Orchestrator branches on plan_selection; the router validates it and only requires scenario_ids for the scenario selection. A recipe without plan_selection defaults to "scenario", so the change is backward-compatible. Both selections now surface the chosen Plan's `name` as a `plan_name` column. Tests: repository default_rows_for (scenario-independent selection, plan_name, non-default excluded); orchestrator default path (one "Default Plans" sheet; empty selection is a recorded failure). 62 passed, pyright --strict clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- applications/ara_export/handler.py | 9 ++- backend/app/exports/router.py | 22 +++++- docs/adr/0065-ara-scenario-export.md | 25 +++++++ domain/scenario_export/scenario_sheet.py | 3 + orchestration/scenario_export_orchestrator.py | 65 ++++++++++++++--- .../scenario_export_postgres_repository.py | 51 +++++++++++-- .../scenario_export_repository.py | 14 ++++ .../test_scenario_export_orchestrator.py | 73 ++++++++++++++++++- ...est_scenario_export_postgres_repository.py | 51 +++++++++++++ 9 files changed, 289 insertions(+), 24 deletions(-) diff --git a/applications/ara_export/handler.py b/applications/ara_export/handler.py index 230441683..b94170caf 100644 --- a/applications/ara_export/handler.py +++ b/applications/ara_export/handler.py @@ -66,10 +66,14 @@ def handler(body: dict[str, Any], context: Any) -> dict[str, Any]: property_ids: list[int] = [int(x) for x in recipe["property_ids"]] recipient_email = str(recipe["recipient_email"]) export_name = str(recipe["export_name"]) + # ADR-0065 plan selection: "scenario" (default) or "default" (each + # home's is_default Plan). Absent on pre-existing recipes → "scenario". + plan_selection = str(recipe.get("plan_selection") or "scenario") logger.info( - "ara_export: starting export for %d properties across %d scenario(s) " - "(subtask=%s)", + "ara_export: starting %s export for %d properties across %d " + "scenario(s) (subtask=%s)", + plan_selection, len(property_ids), len(scenario_ids), trigger.subtask_id, @@ -89,6 +93,7 @@ def handler(body: dict[str, Any], context: Any) -> dict[str, Any]: property_ids=property_ids, recipient_email=recipient_email, export_name=export_name, + plan_selection=plan_selection, ) finally: session.close() diff --git a/backend/app/exports/router.py b/backend/app/exports/router.py index e4d99b9d0..a9ec9afdd 100644 --- a/backend/app/exports/router.py +++ b/backend/app/exports/router.py @@ -92,15 +92,32 @@ async def scenario_export( raw_scenario_ids: Any = config.get("scenario_ids") or [] raw_property_ids: Any = config.get("property_ids") or [] raw_filters: Any = config.get("filters") or {} + # ADR-0065 plan selection: "scenario" (per-Scenario sheets, the default) or + # "default" (one sheet of each home's is_default Plan). Absent → "scenario". + plan_selection: Any = config.get("plan_selection") or "scenario" if not isinstance(portfolio_id, int): raise HTTPException( status_code=400, detail="task.inputs must provide an integer portfolio_id." ) - if not _is_int_list(raw_scenario_ids) or not raw_scenario_ids: + if plan_selection not in ("scenario", "default"): raise HTTPException( status_code=400, - detail="task.inputs scenario_ids must be a non-empty list of integers.", + detail='task.inputs plan_selection must be "scenario" or "default".', + ) + # scenario_ids is required only for a per-Scenario export; the default-plan + # selection is scenario-independent (one default Plan per home). + if plan_selection == "scenario": + if not _is_int_list(raw_scenario_ids) or not raw_scenario_ids: + raise HTTPException( + status_code=400, + detail="task.inputs scenario_ids must be a non-empty list of " + "integers for a scenario export.", + ) + elif raw_scenario_ids and not _is_int_list(raw_scenario_ids): + raise HTTPException( + status_code=400, + detail="task.inputs scenario_ids, if provided, must be integers.", ) if not _is_int_list(raw_property_ids): raise HTTPException( @@ -145,6 +162,7 @@ async def scenario_export( "property_ids": resolved_property_ids, "recipient_email": recipient_email, "export_name": f"portfolio-{portfolio_id}-{body.task_id}", + "plan_selection": plan_selection, } created = tasks.create_export_subtask(body.task_id, subtask_id, recipe) if not created: diff --git a/docs/adr/0065-ara-scenario-export.md b/docs/adr/0065-ara-scenario-export.md index 7cebda333..a4e57c9eb 100644 --- a/docs/adr/0065-ara-scenario-export.md +++ b/docs/adr/0065-ara-scenario-export.md @@ -208,3 +208,28 @@ presigned URL + email (ADR-0059).** the intended shareability, made explicit. - `scripts/data_exports.py`'s query/column logic is de-duplicated into the new modules; the standalone CLI either wraps them or is retired. + +## Addendum — plan-selection option (default plan per home) + +A second **plan selection** was added alongside the original per-Scenario export, +chosen by `plan_selection` on the export request: + +- **`"scenario"` (default, unchanged):** one sheet per requested Scenario, the + freshest Plan per `(property, scenario)`. `scenario_ids` required. +- **`"default"` (new):** a single sheet titled **"Default Plans"** of each home's + **default Plan** (`plan.is_default = TRUE`), which is one-per-property across + all Scenarios (ADR-0012 / ADR-0017) — the portfolio's current state, one row + per home. `scenario_ids` is not required and is ignored. + +The router (`backend/app/exports/router.py`) validates `plan_selection ∈ +{"scenario", "default"}` and only requires a non-empty `scenario_ids` for the +scenario selection; the recipe carries `plan_selection` through to the handler +and orchestrator. The repository gains `default_rows_for(portfolio_id, +property_ids)` — the same read-model and Effective-EPC join as `rows_for`, with +the plan-selection `WHERE scenario_id = …` swapped for `WHERE is_default = TRUE` +(no scenario parameter). A recipe without `plan_selection` (pre-existing tasks) +defaults to `"scenario"`, so the change is backward-compatible. + +**New column — `plan_name`.** Both selections now surface the chosen Plan's +`name` (from `plan.name`) as a `plan_name` column heading the plan post-works +block, so a reader can see which Plan produced each row. diff --git a/domain/scenario_export/scenario_sheet.py b/domain/scenario_export/scenario_sheet.py index ae3860d23..055c91812 100644 --- a/domain/scenario_export/scenario_sheet.py +++ b/domain/scenario_export/scenario_sheet.py @@ -91,6 +91,7 @@ _TOTAL_COLUMNS: tuple[str, ...] = ("total_retrofit_cost",) # The default Plan's post-works figures, taken straight from the persisted Plan # (the SAP calculator's output) with no recompute (ADR-0065). _PLAN_COLUMNS: tuple[str, ...] = ( + "plan_name", "post_sap_points", "post_epc_rating", "cost_of_works", @@ -164,6 +165,7 @@ class PropertyScenarioData: is_expired: Optional[bool] = None current_epc_rating: Optional[str] = None current_sap_points: Optional[float] = None + plan_name: Optional[str] = None post_sap_points: Optional[float] = None post_epc_rating: Optional[str] = None cost_of_works: Optional[float] = None @@ -231,6 +233,7 @@ def _shape_row(prop: PropertyScenarioData) -> dict[str, Any]: row["energy_cost_savings"] = _sum([m.energy_cost_savings for m in prop.measures]) row["total_retrofit_cost"] = _sum([m.estimated_cost for m in prop.measures]) # The default Plan's post-works figures, passed through as persisted. + row["plan_name"] = _blank(prop.plan_name) row["post_sap_points"] = _blank(prop.post_sap_points) row["post_epc_rating"] = _blank(prop.post_epc_rating) row["cost_of_works"] = _blank(prop.cost_of_works) diff --git a/orchestration/scenario_export_orchestrator.py b/orchestration/scenario_export_orchestrator.py index 39d25eccb..657d15b7a 100644 --- a/orchestration/scenario_export_orchestrator.py +++ b/orchestration/scenario_export_orchestrator.py @@ -29,6 +29,9 @@ from repositories.email.email_sender import EmailSender logger = logging.getLogger(__name__) +# The single sheet title for the default-plan selection (ADR-0065). +_DEFAULT_PLAN_SHEET_NAME = "Default Plans" + class ExportRowReader(Protocol): def rows_for( @@ -39,6 +42,13 @@ class ExportRowReader(Protocol): property_ids: Sequence[int], ) -> list[PropertyScenarioData]: ... + def default_rows_for( + self, + *, + portfolio_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: ... + class ScenarioNames(Protocol): def name_for(self, scenario_id: int) -> str: ... @@ -72,6 +82,33 @@ class ScenarioExportOrchestrator: self._url_ttl_seconds = url_ttl_seconds self._key_prefix = key_prefix + def _scenario_sheets( + self, + portfolio_id: int, + scenario_ids: Sequence[int], + property_ids: Sequence[int], + ) -> list[tuple[str, ExportSheet]]: + named_sheets: list[tuple[str, ExportSheet]] = [] + for scenario_id in scenario_ids: + rows = self._rows.rows_for( + portfolio_id=portfolio_id, + scenario_id=scenario_id, + property_ids=property_ids, + ) + named_sheets.append( + (self._scenarios.name_for(scenario_id), shape_scenario_sheet(rows)) + ) + return named_sheets + + def _default_plan_sheets( + self, portfolio_id: int, property_ids: Sequence[int] + ) -> list[tuple[str, ExportSheet]]: + # One sheet: each home's default Plan (`is_default`), across Scenarios. + rows = self._rows.default_rows_for( + portfolio_id=portfolio_id, property_ids=property_ids + ) + return [(_DEFAULT_PLAN_SHEET_NAME, shape_scenario_sheet(rows))] + def run( self, *, @@ -80,26 +117,30 @@ class ScenarioExportOrchestrator: property_ids: Sequence[int], recipient_email: str, export_name: str, + plan_selection: str = "scenario", ) -> ScenarioExportResult: - named_sheets: list[tuple[str, ExportSheet]] = [] - row_count = 0 - for scenario_id in scenario_ids: - rows = self._rows.rows_for( - portfolio_id=portfolio_id, - scenario_id=scenario_id, - property_ids=property_ids, + # ADR-0065 two selections: + # - "scenario": one sheet per requested Scenario, the freshest Plan per + # (property, scenario). + # - "default": ONE sheet of each home's default Plan (`is_default`), + # one-per-property across all Scenarios — the portfolio's current + # state. `scenario_ids` is ignored. + if plan_selection == "default": + named_sheets = self._default_plan_sheets(portfolio_id, property_ids) + else: + named_sheets = self._scenario_sheets( + portfolio_id, scenario_ids, property_ids ) - sheet = shape_scenario_sheet(rows) - named_sheets.append((self._scenarios.name_for(scenario_id), sheet)) - row_count += len(sheet.rows) + row_count = sum(len(sheet.rows) for _, sheet in named_sheets) if row_count == 0: - # No Property had a default Plan under any requested Scenario (model - # A) — a recorded failure, never an empty workbook emailed (ADR-0065). + # No Property had a Plan under the selection (model A) — a recorded + # failure, never an empty workbook emailed (ADR-0065). raise SubTaskFailure( "the scenario export selection produced no exportable rows", details={ "portfolio_id": portfolio_id, + "plan_selection": plan_selection, "scenario_ids": list(scenario_ids), "property_count": len(property_ids), }, diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index c17c220d9..20fac6bdf 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -29,12 +29,13 @@ _ROWS_SQL = text( "SELECT p.id, p.landlord_property_id, p.uprn, p.address, p.postcode," " np.post_sap_points, np.post_epc_rating, np.cost_of_works," " np.contingency_cost, np.co2_savings, np.energy_bill_savings," - " np.energy_consumption_savings, np.valuation_increase" + " np.energy_consumption_savings, np.valuation_increase, np.plan_name" " FROM property p" " JOIN (" " SELECT DISTINCT ON (property_id) property_id, post_sap_points," " post_epc_rating, cost_of_works, contingency_cost, co2_savings," - " energy_bill_savings, energy_consumption_savings, valuation_increase" + " energy_bill_savings, energy_consumption_savings, valuation_increase," + " name AS plan_name" " FROM plan" " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" " AND property_id = ANY(:property_ids)" @@ -65,6 +66,21 @@ _MEASURES_SQL = text( " AND r.already_installed = FALSE" ) +# DEFAULT-PLAN variants (ADR-0065 "default plan per home" selection). Each home's +# ONE default Plan (`is_default = TRUE`), which is one-per-property across all +# Scenarios (ADR-0012 / ADR-0017) — so there is no `scenario_id` filter. The +# `DISTINCT ON (property_id) … created_at DESC` guard is retained as belt-and- +# braces against a transient double-default during a re-baseline write; steady +# state has exactly one. Otherwise identical to the scenario SQL above. +_DEFAULT_ROWS_SQL = text( + str(_ROWS_SQL) + .replace("scenario_id = :scenario_id", "is_default = TRUE") +) +_DEFAULT_MEASURES_SQL = text( + str(_MEASURES_SQL) + .replace("scenario_id = :scenario_id", "is_default = TRUE") +) + # The Effective-EPC descriptive picture (ADR-0065). Header facts and current # performance are matched by UPRN (the latest epc_property for the source), # because the PasHub re-fetch lands records with property_id = NULL — a @@ -221,9 +237,31 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): "scenario_id": scenario_id, "property_ids": list(property_ids), } - measures = self._measures_by_property(params) + return self._assemble_rows(_ROWS_SQL, _MEASURES_SQL, params) + + def default_rows_for( + self, + *, + portfolio_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + if not property_ids: + return [] + # No scenario_id: the default-plan SQL selects on `is_default = TRUE`. + params: dict[str, Any] = { + "portfolio_id": portfolio_id, + "property_ids": list(property_ids), + } + return self._assemble_rows( + _DEFAULT_ROWS_SQL, _DEFAULT_MEASURES_SQL, params + ) + + def _assemble_rows( + self, rows_sql: Any, measures_sql: Any, params: dict[str, Any] + ) -> list[PropertyScenarioData]: + measures = self._measures_by_property(params, measures_sql) epc = self._effective_epc_by_property(params) - result = self._session.connection().execute(_ROWS_SQL, params) + result = self._session.connection().execute(rows_sql, params) rows: list[PropertyScenarioData] = [] for row in result.all(): facts = epc.get(row[0], {}) @@ -242,6 +280,7 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): energy_bill_savings=_float(row[10]), energy_consumption_savings=_float(row[11]), valuation_increase=_float(row[12]), + plan_name=_str(row[13]), property_type=facts.get("property_type"), built_form=facts.get("built_form"), property_age_band=facts.get("property_age_band"), @@ -264,9 +303,9 @@ class ScenarioExportPostgresRepository(ScenarioExportRepository): return rows def _measures_by_property( - self, params: dict[str, Any] + self, params: dict[str, Any], measures_sql: Any = _MEASURES_SQL ) -> dict[int, list[ExportMeasure]]: - result = self._session.connection().execute(_MEASURES_SQL, params) + result = self._session.connection().execute(measures_sql, params) by_property: dict[int, list[ExportMeasure]] = {} for row in result.all(): by_property.setdefault(row[0], []).append( diff --git a/repositories/scenario_export/scenario_export_repository.py b/repositories/scenario_export/scenario_export_repository.py index dfc9cb113..a3cae4e7d 100644 --- a/repositories/scenario_export/scenario_export_repository.py +++ b/repositories/scenario_export/scenario_export_repository.py @@ -23,3 +23,17 @@ class ScenarioExportRepository(ABC): selected measures. A Property with no default Plan for the Scenario is absent (scenario-scoped, model A).""" ... + + @abstractmethod + def default_rows_for( + self, + *, + portfolio_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + """The export rows for the Properties in ``property_ids`` selected by + each home's **default Plan** (``plan.is_default = TRUE``), which is + one-per-property across all Scenarios (ADR-0012 / ADR-0017). Unlike + ``rows_for`` this is NOT scenario-scoped — it is the portfolio's current + state, one row per home. A Property with no default Plan is absent.""" + ... diff --git a/tests/orchestration/test_scenario_export_orchestrator.py b/tests/orchestration/test_scenario_export_orchestrator.py index 86ce864ea..26c678da3 100644 --- a/tests/orchestration/test_scenario_export_orchestrator.py +++ b/tests/orchestration/test_scenario_export_orchestrator.py @@ -20,10 +20,15 @@ EXPORTS_BUCKET = "domna-exports-dev" class _FakeRows: - """Returns preset export rows per scenario id.""" + """Returns preset export rows per scenario id, and a preset default-plan set.""" - def __init__(self, by_scenario: dict[int, list[PropertyScenarioData]]) -> None: + def __init__( + self, + by_scenario: dict[int, list[PropertyScenarioData]], + default_rows: Optional[list[PropertyScenarioData]] = None, + ) -> None: self._by_scenario = by_scenario + self._default_rows = default_rows or [] def rows_for( self, @@ -34,6 +39,14 @@ class _FakeRows: ) -> list[PropertyScenarioData]: return self._by_scenario.get(scenario_id, []) + def default_rows_for( + self, + *, + portfolio_id: int, + property_ids: Sequence[int], + ) -> list[PropertyScenarioData]: + return self._default_rows + class _FakeScenarioNames: def __init__(self, names: dict[int, str]) -> None: @@ -182,3 +195,59 @@ def test_a_selection_that_yields_no_rows_is_a_recorded_failure( export_name="portfolio-100", ) assert email.sent == [] + + +def test_default_plan_selection_builds_one_sheet_from_each_homes_default_plan( + exports: S3Client, +) -> None: + # arrange — plan_selection="default" ignores scenario_ids and reads each + # home's default Plan (ADR-0065). The fake returns two default rows and + # would raise if `rows_for` (the scenario path) were called instead. + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({}, default_rows=[_row(1), _row(2)]), + scenarios=_FakeScenarioNames({}), + exports=exports, + email=_RecordingEmail(), + url_ttl_seconds=3600, + ) + + # act — no scenario_ids needed for the default-plan selection. + result = orchestrator.run( + portfolio_id=100, + scenario_ids=[], + property_ids=[1, 2], + recipient_email="user@example.com", + export_name="portfolio-100-default", + plan_selection="default", + ) + + # assert — exactly one sheet, titled "Default Plans", with both homes' rows. + assert result.sheet_count == 1 + assert result.row_count == 2 + workbook = load_workbook(BytesIO(exports.get_object(result.export_s3_key))) + assert workbook.sheetnames == ["Default Plans"] + + +def test_default_selection_with_no_default_plans_is_a_recorded_failure( + exports: S3Client, +) -> None: + # arrange — no home has a default Plan. + orchestrator = ScenarioExportOrchestrator( + rows=_FakeRows({}, default_rows=[]), + scenarios=_FakeScenarioNames({}), + exports=exports, + email=_RecordingEmail(), + url_ttl_seconds=3600, + ) + + # act / assert — a selection that yields no rows is a failure, never an + # empty workbook emailed. + with pytest.raises(SubTaskFailure): + orchestrator.run( + portfolio_id=100, + scenario_ids=[], + property_ids=[1, 2], + recipient_email="user@example.com", + export_name="portfolio-100-default", + plan_selection="default", + ) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index 0210502ad..63d5744df 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -662,3 +662,54 @@ def test_falls_back_to_the_predicted_epc_when_there_is_no_lodged_epc( row = rows[0] assert row.property_type == "Bungalow" assert row.walls == "Predicted solid brick wall" + + +def test_default_rows_for_selects_each_homes_default_plan_across_scenarios( + db_engine: Engine, +) -> None: + # arrange — two homes, each with its default Plan under a DIFFERENT scenario + # (property 1 → scenario 5, property 2 → scenario 9), plus a non-default Plan + # on property 1 under a third scenario. The default-plan selection is + # scenario-independent (ADR-0012 / ADR-0017): it must return exactly the two + # `is_default=True` Plans, carrying the Plan name, and ignore the non-default. + with Session(db_engine) as session: + session.add( + PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=1) + ) + session.add( + PropertyRow(id=2, portfolio_id=100, landlord_property_id="LP2", uprn=2) + ) + session.add( + PlanModel( + portfolio_id=100, property_id=1, scenario_id=5, is_default=True, + name="Fabric First", post_sap_points=78.0, + ) + ) + session.add( + PlanModel( + portfolio_id=100, property_id=1, scenario_id=7, is_default=False, + name="Low Carbon", post_sap_points=81.0, + ) + ) + session.add( + PlanModel( + portfolio_id=100, property_id=2, scenario_id=9, is_default=True, + name="Whole House", post_sap_points=69.0, + ) + ) + session.commit() + + # act — no scenario_id. + with Session(db_engine) as session: + rows = ScenarioExportPostgresRepository(session).default_rows_for( + portfolio_id=100, property_ids=[1, 2] + ) + + # assert — one row per home, the default Plan, with its name; the non-default + # "Low Carbon" (81.0) never appears. + by_property = {r.property_id: r for r in rows} + assert set(by_property) == {1, 2} + assert by_property[1].plan_name == "Fabric First" + assert by_property[1].post_sap_points == 78.0 + assert by_property[2].plan_name == "Whole House" + assert by_property[2].post_sap_points == 69.0 From b79d16913e224ab6581f5029f1859a4bb25bd9a0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Jul 2026 09:35:57 +0000 Subject: [PATCH 46/46] plan_name column sources the source Scenario name, not plan.name `plan.name` is a freeform, user-entered label that is unset on modelling-generated Plans (0/205 populated on a real portfolio). The meaningful "name of the plan chosen" is the Scenario the home's default Plan came from, so `plan_name` now reads `scenario.name` via `plan.scenario_id` (LEFT JOIN scenario) on both export selections. On the default-plan export this labels each home with its source scenario (e.g. "Fabric Only" / "ASHP + PV"); on the scenario export it equals that sheet's scenario. Repository test seeds scenarios and asserts the source-scenario name; 62 tests pass, pyright --strict clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/adr/0065-ara-scenario-export.md | 8 +++--- .../scenario_export_postgres_repository.py | 5 ++-- ...est_scenario_export_postgres_repository.py | 25 +++++++++++++------ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/adr/0065-ara-scenario-export.md b/docs/adr/0065-ara-scenario-export.md index a4e57c9eb..cf934ee46 100644 --- a/docs/adr/0065-ara-scenario-export.md +++ b/docs/adr/0065-ara-scenario-export.md @@ -230,6 +230,8 @@ the plan-selection `WHERE scenario_id = …` swapped for `WHERE is_default = TRU (no scenario parameter). A recipe without `plan_selection` (pre-existing tasks) defaults to `"scenario"`, so the change is backward-compatible. -**New column — `plan_name`.** Both selections now surface the chosen Plan's -`name` (from `plan.name`) as a `plan_name` column heading the plan post-works -block, so a reader can see which Plan produced each row. +**New column — `plan_name`.** Both selections surface the **source Scenario's +name** (`scenario.name`, via `plan.scenario_id`) as a `plan_name` column heading +the plan post-works block, so a reader sees which Scenario each home's Plan came +from (e.g. "Fabric Only" / "ASHP + PV"). The freeform `plan.name` is unset on +modelling-generated Plans, so the Scenario name is the meaningful identifier. diff --git a/repositories/scenario_export/scenario_export_postgres_repository.py b/repositories/scenario_export/scenario_export_postgres_repository.py index 20fac6bdf..03b1e91bb 100644 --- a/repositories/scenario_export/scenario_export_postgres_repository.py +++ b/repositories/scenario_export/scenario_export_postgres_repository.py @@ -29,18 +29,19 @@ _ROWS_SQL = text( "SELECT p.id, p.landlord_property_id, p.uprn, p.address, p.postcode," " np.post_sap_points, np.post_epc_rating, np.cost_of_works," " np.contingency_cost, np.co2_savings, np.energy_bill_savings," - " np.energy_consumption_savings, np.valuation_increase, np.plan_name" + " np.energy_consumption_savings, np.valuation_increase, sc.name AS plan_name" " FROM property p" " JOIN (" " SELECT DISTINCT ON (property_id) property_id, post_sap_points," " post_epc_rating, cost_of_works, contingency_cost, co2_savings," " energy_bill_savings, energy_consumption_savings, valuation_increase," - " name AS plan_name" + " scenario_id" " FROM plan" " WHERE portfolio_id = :portfolio_id AND scenario_id = :scenario_id" " AND property_id = ANY(:property_ids)" " ORDER BY property_id, created_at DESC, id DESC" " ) np ON np.property_id = p.id" + " LEFT JOIN scenario sc ON sc.id = np.scenario_id" " WHERE p.portfolio_id = :portfolio_id" " ORDER BY p.id" ) diff --git a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py index 63d5744df..e0a4f825d 100644 --- a/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py +++ b/tests/repositories/scenario_export/test_scenario_export_postgres_repository.py @@ -10,6 +10,7 @@ from sqlalchemy import Engine from sqlmodel import Session from datatypes.epc.domain.epc import Epc +from domain.modelling.portfolio_goal import PortfolioGoal from infrastructure.postgres.epc_property_table import ( # registers epc tables EpcBuildingPartModel, EpcEnergyElementModel, @@ -20,6 +21,7 @@ from infrastructure.postgres.epc_property_table import ( # registers epc tables from infrastructure.postgres.modelling import ( # registers `plan` + `recommendation` PlanModel, RecommendationModel, + ScenarioModel, ) from infrastructure.postgres.product_table import MaterialRow # registers `material` from infrastructure.postgres.property_override_table import ( # registers overrides @@ -671,8 +673,15 @@ def test_default_rows_for_selects_each_homes_default_plan_across_scenarios( # (property 1 → scenario 5, property 2 → scenario 9), plus a non-default Plan # on property 1 under a third scenario. The default-plan selection is # scenario-independent (ADR-0012 / ADR-0017): it must return exactly the two - # `is_default=True` Plans, carrying the Plan name, and ignore the non-default. + # `is_default=True` Plans, and `plan_name` carries the SOURCE SCENARIO's name + # (not the freeform `plan.name`, which is unset on modelling-generated Plans). with Session(db_engine) as session: + session.add(ScenarioModel(id=5, portfolio_id=100, name="Fabric Only", + goal=PortfolioGoal.INCREASING_EPC, goal_value="C")) + session.add(ScenarioModel(id=7, portfolio_id=100, name="Low Carbon", + goal=PortfolioGoal.INCREASING_EPC, goal_value="C")) + session.add(ScenarioModel(id=9, portfolio_id=100, name="ASHP + PV", + goal=PortfolioGoal.INCREASING_EPC, goal_value="C")) session.add( PropertyRow(id=1, portfolio_id=100, landlord_property_id="LP1", uprn=1) ) @@ -682,19 +691,19 @@ def test_default_rows_for_selects_each_homes_default_plan_across_scenarios( session.add( PlanModel( portfolio_id=100, property_id=1, scenario_id=5, is_default=True, - name="Fabric First", post_sap_points=78.0, + post_sap_points=78.0, ) ) session.add( PlanModel( portfolio_id=100, property_id=1, scenario_id=7, is_default=False, - name="Low Carbon", post_sap_points=81.0, + post_sap_points=81.0, ) ) session.add( PlanModel( portfolio_id=100, property_id=2, scenario_id=9, is_default=True, - name="Whole House", post_sap_points=69.0, + post_sap_points=69.0, ) ) session.commit() @@ -705,11 +714,11 @@ def test_default_rows_for_selects_each_homes_default_plan_across_scenarios( portfolio_id=100, property_ids=[1, 2] ) - # assert — one row per home, the default Plan, with its name; the non-default - # "Low Carbon" (81.0) never appears. + # assert — one row per home, the default Plan, with its source scenario name; + # the non-default plan under "Low Carbon" (81.0) never appears. by_property = {r.property_id: r for r in rows} assert set(by_property) == {1, 2} - assert by_property[1].plan_name == "Fabric First" + assert by_property[1].plan_name == "Fabric Only" assert by_property[1].post_sap_points == 78.0 - assert by_property[2].plan_name == "Whole House" + assert by_property[2].plan_name == "ASHP + PV" assert by_property[2].post_sap_points == 69.0