feat(scripts): one-pass e2e run — baseline + plans + prediction + recovery

Make run_modelling_e2e the single script that does everything for a portfolio,
so the 291-property run needs one invocation with per-property recovery (no
all-or-nothing chunking):

- On --persist, a lodged-EPC Property now also gets its Baseline Performance
  row written via PropertyBaselineOrchestrator (per Property, so one bad cert
  does not abort the batch). Predicted (EPC-less) Properties have no lodged
  figures, so they get a Plan but no baseline row.
- The run CSV gains api_sap (register) vs baseline_sap (calculator) + sap_delta,
  so calculator-vs-API divergence is reviewable per property.

Fill the off-catalogue overlay for the measures the live material catalogue
cannot price, so they stop crashing the run:

- double_glazing (£550/window) and secondary_glazing (£400/window): priced
  per window (the generator multiplies by single-glazed window count, matching
  the legacy window_glazing). Grounded in 2025/26 UK installed costs; per-window
  is the right unit for windows (fixed per-unit install dominates) — per-m2 fits
  walls/floors, not glazing.
- gas_boiler_upgrade / system_tune_up / system_tune_up_zoned: these are priced
  off the heating rate sheet (Products()), with get() reading the catalogue only
  for an id — so the overlay entry exists to satisfy that lookup (material_id
  stays None, as with ASHP); the rate sheet remains authoritative.

Validated on a 12-property sample (incl. a secondary-glazing case and a
SAP-Schema-16.2 cert): 12/12 baseline rows + plans, 0 errors.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

repositories/product/off_catalogue_costs.json

@@ -1,3 +1,8 @@
 {
-  "secondary_heating_removal": { "unit_cost_per_m2": 270.0 }
+  "secondary_heating_removal": { "unit_cost_per_m2": 270.0 },
+  "double_glazing": { "unit_cost_per_m2": 550.0 },
+  "secondary_glazing": { "unit_cost_per_m2": 400.0 },
+  "gas_boiler_upgrade": { "unit_cost_per_m2": 3000.0 },
+  "system_tune_up": { "unit_cost_per_m2": 500.0 },
+  "system_tune_up_zoned": { "unit_cost_per_m2": 900.0 }
 }

scripts/run_modelling_e2e.py

@@ -9,7 +9,12 @@ from S3, and fetches Google Solar — then runs the Modelling stage (every
 Recommendation Generator → the Optimiser → a costed, attributed Plan). The same
 local computation runs whether or not you store the result: by default it
 persists **nothing** (the run is for inspecting recommendations); pass
-`--persist` to write the inputs + the Plan to the DB.
+`--persist` to write the inputs + the Plan to the DB. With `--persist`, a
+lodged-EPC Property **also** gets its Baseline Performance row written (lodged
+vs calculator-effective SAP + the bill block) via the same orchestrator the
+first-run pipeline uses — run per Property so one bad cert does not abort the
+batch. A predicted (EPC-less) Property has no lodged figures, so it gets a Plan
+but no baseline row.
 
 To keep the inspected recommendations identical to what gets stored, **both
 modes price against the live ``material`` catalogue (read-only)** and model
@@ -21,12 +26,9 @@ the run synthesises an Increasing-EPC-to-``--goal`` Scenario with no exclusions.
 `--measures` / `--exclude-measures` are optional overlays layered on top of the
 Scenario's own exclusions.
 
-KNOWN GOTCHA: the live ``material.type`` enum does not yet carry
-``secondary_heating_removal``, so a Property with a lodged secondary heater
-crashes the catalogue read for that measure. Until the catalogue stocks it, pass
-``--exclude-measures secondary_heating_removal`` (an ASHP row is also absent, but
-ASHP is priced off the rate sheet so it degrades to ``material_id=None`` rather
-than crashing — no flag needed).
+``secondary_heating_removal`` is priced from the committed off-catalogue JSON
+overlay (the live ``material.type`` enum cannot carry it), so no exclusion flag
+is needed. ASHP is priced off the rate sheet (``material_id=None``), also fine.
 
 Config: loads `backend/.env` for the DB creds (`DB_*`), the EPC API token
 (`OPEN_EPC_API_TOKEN` — the Bearer token for the new gov API), the Google Solar
@@ -111,6 +113,16 @@ from repositories.property.override_backed_prediction_attributes_reader import (
     OverrideBackedPredictionAttributesReader,
 )
 from repositories.postgres_unit_of_work import PostgresUnitOfWork  # noqa: E402
+from orchestration.property_baseline_orchestrator import (  # noqa: E402
+    PropertyBaselineOrchestrator,
+)
+from domain.property_baseline.calculator_rebaseliner import (  # noqa: E402
+    CalculatorRebaseliner,
+)
+from domain.sap10_calculator.calculator import Sap10Calculator  # noqa: E402
+from repositories.fuel_rates.fuel_rates_static_file_repository import (  # noqa: E402
+    FuelRatesStaticFileRepository,
+)
 from repositories.scenario.scenario_postgres_repository import (  # noqa: E402
     ScenarioPostgresRepository,
 )
@@ -520,6 +532,18 @@ def main() -> None:
     # batch so both store and no-store runs price against the same DB rows.
     catalogue_session = Session(engine)
     products = catalogue_with_off_catalogue_overrides(catalogue_session)
+    # When persisting, a lodged-EPC Property also gets a Baseline Performance row
+    # via the production orchestrator (lodged-vs-calculator SAP, the bill block) —
+    # the same establish-and-persist the first-run pipeline runs, here per Property
+    # so one bad cert doesn't abort the batch. Predicted Properties have no lodged
+    # figures, so they are skipped below.
+    baseline_orchestrator: Optional[PropertyBaselineOrchestrator] = None
+    if args.persist:
+        baseline_orchestrator = PropertyBaselineOrchestrator(
+            unit_of_work=lambda: PostgresUnitOfWork(lambda: Session(engine)),
+            rebaseliner=CalculatorRebaseliner(Sap10Calculator()),
+            fuel_rates=FuelRatesStaticFileRepository(),
+        )
     scenario: Optional[Scenario] = (
         _scenario_for(catalogue_session, args.scenario_id)
         if args.scenario_id is not None
@@ -546,7 +570,8 @@ def main() -> None:
 
     md_lines: list[str] = [f"# Modelling recommendations ({target}, {measures_note})\n"]
     csv_rows: list[str] = [
-        "property_id,uprn,baseline_sap,post_sap,measures,measure_types,cost_of_works"
+        "property_id,uprn,api_sap,baseline_sap,sap_delta,post_sap,measures,"
+        "measure_types,cost_of_works"
     ]
     candidate_csv_rows: list[str] = [
         "property_id,uprn,surface,measure_type,cost_total,contingency_rate,"
@@ -690,6 +715,12 @@ def main() -> None:
                     solar_insights=solar_insights,
                     plan=plan,
                 )
+                # A lodged EPC also gets its Baseline Performance persisted
+                # (reads the EPC just saved above). Predicted Properties have no
+                # lodged figures to baseline, so they are skipped.
+                if epc is not None:
+                    assert baseline_orchestrator is not None
+                    baseline_orchestrator.run([property_id])
         except (
             Exception
         ) as error:  # noqa: BLE001 — one bad property must not stop the run
@@ -702,7 +733,7 @@ def main() -> None:
             line = f"property {property_id} (uprn {uprn}): ERROR — {type(error).__name__}: {error}"
             print(line + "\n")
             md_lines.append(f"## Property {property_id}\n\n`{line}`\n")
-            csv_rows.append(f"{property_id},{uprn or ''},,,,ERROR,")
+            csv_rows.append(f"{property_id},{uprn or ''},,,,,,ERROR,")
             continue
 
         measure_types = [m.measure_type for m in plan.measures]
@@ -737,8 +768,16 @@ def main() -> None:
         md_lines.append("**All candidate measures (cost per measure)**\n")
         md_lines.extend(candidate_lines)
         md_lines.append("")
+        # api_sap is the lodged/register SAP (off the cert); a predicted Property
+        # has none, so it and the delta are left blank. baseline_sap is the
+        # calculator's score on the Effective EPC — the two whose divergence the
+        # run is for reviewing (mirrors lodged vs effective in the baseline table).
+        api_sap: Optional[int] = epc.energy_rating_current if epc is not None else None
+        calc_sap: float = plan.baseline.sap_continuous
+        api_sap_cell = "" if api_sap is None else str(api_sap)
+        sap_delta_cell = "" if api_sap is None else f"{calc_sap - api_sap:.2f}"
         csv_rows.append(
-            f"{property_id},{uprn},{plan.baseline.sap_continuous:.2f},"
+            f"{property_id},{uprn},{api_sap_cell},{calc_sap:.2f},{sap_delta_cell},"
             f"{plan.post_sap_continuous:.2f},{len(plan.measures)},"
             f"{'|'.join(measure_types)},{plan.cost_of_works:.0f}"
         )

tests/repositories/product/test_composite_product_repository.py

@@ -99,3 +99,11 @@ def test_committed_override_file_prices_secondary_heating_removal() -> None:
 
     assert abs(product.unit_cost_per_m2 - 270.0) <= 1e-9
     assert abs(product.contingency_rate - 0.25) <= 1e-9
+
+
+def test_committed_override_file_prices_the_glazing_gaps() -> None:
+    # Glazing is absent from the live catalogue (no active rows), so the overlay
+    # carries per-window costs the generator uses directly (count x unit cost).
+    repo = ProductJsonRepository(OFF_CATALOGUE_COSTS_PATH)
+    assert abs(repo.get("double_glazing").unit_cost_per_m2 - 550.0) <= 1e-9
+    assert abs(repo.get("secondary_glazing").unit_cost_per_m2 - 400.0) <= 1e-9