feat(modelling): price secondary-heating-removal from an off-catalogue overlay

The FE-owned `material.type` pgEnum cannot carry `secondary_heating_removal`,
so pricing it through the DB catalogue raises a DataError that poisons the
session — the modelling pipeline crashed on any property with a lodged
secondary heater unless the measure was excluded on the Scenario.

Realise the `ProductRepository` docstring's intent (DB catalogue today, a JSON
file for costs the ETL does not yet supply, behind the same port): add a
`CompositeProductRepository` that resolves an override source first, then the
catalogue. Checking the override first keeps that Measure Type away from the DB
entirely; every other type misses the override and falls through unchanged.

- off_catalogue_costs.json prices it at £270 flat per-dwelling — the legacy
  `Costs.heater_removal` ported to the new flat model (ADR-0028):
  (£25 + £200 baseline) x 1.2 VAT, for the single fixed secondary a cert lodges.
  Contingency (0.25) is joined from config, not the file.
- Wire the composite into PostgresUnitOfWork.product and run_modelling_e2e, so
  the first-run pipeline and the local runner both honour the overlay.
- Integration test: drop the unrealistic seeded secondary_heating_removal DB
  rows (the pgEnum can't hold the type) and assert it is JSON-sourced
  (material_id is None, cost £270) end-to-end through a real Unit of Work.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

repositories/postgres_unit_of_work.py

@@ -11,8 +11,8 @@ from repositories.property_baseline.property_baseline_postgres_repository import
 )
 from repositories.epc.epc_postgres_repository import EpcPostgresRepository
 from repositories.plan.plan_postgres_repository import PlanPostgresRepository
-from repositories.product.product_postgres_repository import (
-    ProductPostgresRepository,
+from repositories.product.composite_product_repository import (
+    catalogue_with_off_catalogue_overrides,
 )
 from repositories.property.property_postgres_repository import (
     PropertyPostgresRepository,
@@ -49,7 +49,7 @@ class PostgresUnitOfWork(UnitOfWork):
         self.spatial = spatial_repo
         self.property_baseline = PropertyBaselinePostgresRepository(self._session)
         self.scenario = ScenarioPostgresRepository(self._session)
-        self.product = ProductPostgresRepository(self._session)
+        self.product = catalogue_with_off_catalogue_overrides(self._session)
         self.plan = PlanPostgresRepository(self._session)
         return self
 

repositories/product/composite_product_repository.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from sqlmodel import Session
+
+from domain.modelling.product import Product
+from repositories.product.product_json_repository import ProductJsonRepository
+from repositories.product.product_postgres_repository import (
+    ProductPostgresRepository,
+)
+from repositories.product.product_repository import ProductRepository
+
+# The committed off-catalogue cost overlay — Measure Types the live ``material``
+# catalogue cannot supply, priced from a static JSON until the ETL stocks them.
+OFF_CATALOGUE_COSTS_PATH: Path = Path(__file__).resolve().parent / "off_catalogue_costs.json"
+
+
+class CompositeProductRepository(ProductRepository):
+    """Resolves a Product from an ``override`` source first, then a ``fallback``.
+
+    The override holds off-catalogue costs the live ``material`` catalogue cannot
+    supply — today ``secondary_heating_removal``, whose Measure Type the FE-owned
+    ``material.type`` pgEnum does not carry. Querying that value against the DB
+    raises a ``DataError`` that poisons the session's transaction (it is not a
+    benign ``ProductNotFound``), so checking the override **first** keeps that
+    Measure Type away from the catalogue entirely. Every other Measure Type
+    misses the override and falls through to the catalogue unchanged.
+    """
+
+    def __init__(
+        self, override: ProductRepository, fallback: ProductRepository
+    ) -> None:
+        self._override = override
+        self._fallback = fallback
+
+    def get(self, measure_type: str) -> Product:
+        product = self._override.get_optional(measure_type)
+        if product is not None:
+            return product
+        return self._fallback.get(measure_type)
+
+
+def catalogue_with_off_catalogue_overrides(session: Session) -> ProductRepository:
+    """The live ``material`` catalogue with the committed off-catalogue JSON costs
+    layered on top — the single composition point both the pipeline's Unit of
+    Work and the local e2e runner price through."""
+    return CompositeProductRepository(
+        override=ProductJsonRepository(OFF_CATALOGUE_COSTS_PATH),
+        fallback=ProductPostgresRepository(session),
+    )

repositories/product/off_catalogue_costs.json

@@ -0,0 +1,3 @@
+{
+  "secondary_heating_removal": { "unit_cost_per_m2": 270.0 }
+}

scripts/run_modelling_e2e.py

@@ -81,8 +81,8 @@ from infrastructure.solar.google_solar_api_client import (  # noqa: E402
 from repositories.geospatial.geospatial_s3_repository import (  # noqa: E402
     GeospatialS3Repository,
 )
-from repositories.product.product_postgres_repository import (  # noqa: E402
-    ProductPostgresRepository,
+from repositories.product.composite_product_repository import (  # noqa: E402
+    catalogue_with_off_catalogue_overrides,
 )
 from repositories.postgres_unit_of_work import PostgresUnitOfWork  # noqa: E402
 from repositories.scenario.scenario_postgres_repository import (  # noqa: E402
@@ -361,7 +361,7 @@ def main() -> None:
     # One read-only session for the live `material` catalogue, reused across the
     # batch so both store and no-store runs price against the same DB rows.
     catalogue_session = Session(engine)
-    products = ProductPostgresRepository(catalogue_session)
+    products = catalogue_with_off_catalogue_overrides(catalogue_session)
     scenario: Optional[Scenario] = (
         _scenario_for(catalogue_session, args.scenario_id)
         if args.scenario_id is not None

tests/orchestration/test_ara_first_run_pipeline_integration.py

@@ -192,14 +192,8 @@ def test_first_run_baselines_through_repos_and_is_idempotent_on_rerun(
                     is_active=True,
                     description="Zoned heating controls + cylinder tune-up",
                 ),
-                MaterialRow(
-                    id=9,
-                    type="secondary_heating_removal",
-                    total_cost=250.0,
-                    cost_unit="gbp_per_unit",
-                    is_active=True,
-                    description="Secondary heating removal",
-                ),
+                # secondary_heating_removal is off-catalogue (priced from the
+                # JSON overlay, not the pgEnum-constrained material table).
             ]
         )
         session.commit()
@@ -324,14 +318,10 @@ def test_modelling_optimises_and_persists_a_multi_measure_plan(
                     is_active=True,
                     description="LED bulb",
                 ),
-                MaterialRow(
-                    id=9,
-                    type="secondary_heating_removal",
-                    total_cost=250.0,
-                    cost_unit="gbp_per_unit",
-                    is_active=True,
-                    description="Secondary heating removal",
-                ),
+                # No secondary_heating_removal row: the FE-owned ``material.type``
+                # pgEnum cannot carry that Measure Type, so it is priced from the
+                # committed off-catalogue JSON overlay (£270, no material_id) that
+                # the Unit of Work layers over the catalogue, not from the DB.
             ]
         )
         session.commit()
@@ -402,13 +392,17 @@ def test_modelling_optimises_and_persists_a_multi_measure_plan(
         "air_source_heat_pump",
         "secondary_heating_removal",
     }
-    # Each persisted measure carries the catalogue id of the Product it installs
+    # Each catalogue-sourced measure carries the id of the Product it installs
     # (the MaterialRow ids seeded above), replacing the retired
     # recommendation_materials BOM with a single material_id on the row.
     assert by_type["air_source_heat_pump"].material_id == 5
     assert by_type["suspended_floor_insulation"].material_id == 2
     assert by_type["low_energy_lighting"].material_id == 4
-    assert by_type["secondary_heating_removal"].material_id == 9
+    # Secondary heating removal is priced from the off-catalogue JSON overlay
+    # (£270 flat per-dwelling, ADR-0028), so it carries no catalogue material_id.
+    assert by_type["secondary_heating_removal"].material_id is None
+    assert by_type["secondary_heating_removal"].estimated_cost is not None
+    assert abs(by_type["secondary_heating_removal"].estimated_cost - 270.0) <= 1e-6
     for rec in rec_rows:
         assert rec.default is True
         assert rec.already_installed is False
@@ -498,14 +492,8 @@ def test_modelling_recommends_nothing_when_already_at_the_target_band(
                     is_active=True,
                     description="LED bulb",
                 ),
-                MaterialRow(
-                    id=9,
-                    type="secondary_heating_removal",
-                    total_cost=250.0,
-                    cost_unit="gbp_per_unit",
-                    is_active=True,
-                    description="Secondary heating removal",
-                ),
+                # secondary_heating_removal is off-catalogue (priced from the
+                # JSON overlay, not the pgEnum-constrained material table).
             ]
         )
         session.commit()

tests/repositories/product/test_composite_product_repository.py

@@ -0,0 +1,101 @@
+"""Behaviour of the composite ProductRepository: an off-catalogue override
+source checked first, the live catalogue as fallback. The override holds costs
+the ``material`` catalogue cannot supply (today ``secondary_heating_removal``,
+whose Measure Type the ``material.type`` pgEnum does not carry). Resolving it
+from the override keeps that Measure Type away from the DB entirely, while every
+other Measure Type falls through to the catalogue unchanged."""
+
+import pytest
+
+from domain.modelling.product import Product
+from repositories.product.composite_product_repository import (
+    OFF_CATALOGUE_COSTS_PATH,
+    CompositeProductRepository,
+)
+from repositories.product.product_json_repository import ProductJsonRepository
+from repositories.product.product_repository import ProductNotFound, ProductRepository
+
+
+class _StaticRepo(ProductRepository):
+    """A ProductRepository over a fixed measure-type → Product map."""
+
+    def __init__(self, products: dict[str, Product]) -> None:
+        self._products = products
+
+    def get(self, measure_type: str) -> Product:
+        product = self._products.get(measure_type)
+        if product is None:
+            raise ProductNotFound(f"no product for measure type {measure_type!r}")
+        return product
+
+
+class _ExplodingRepo(ProductRepository):
+    """A fallback that must never be reached — any read fails the test loudly,
+    standing in for the catalogue whose pgEnum query would poison the session."""
+
+    def get(self, measure_type: str) -> Product:
+        raise AssertionError(
+            f"fallback was queried for {measure_type!r}; the override should have "
+            "resolved it without touching the catalogue"
+        )
+
+
+def _product(measure_type: str, unit_cost: float) -> Product:
+    return Product(
+        measure_type=measure_type, unit_cost_per_m2=unit_cost, contingency_rate=0.25
+    )
+
+
+def test_get_returns_the_override_without_touching_the_fallback() -> None:
+    # Arrange — the override supplies the off-catalogue cost; the fallback would
+    # raise if consulted (mirroring the live pgEnum DataError on this type).
+    override = _StaticRepo(
+        {"secondary_heating_removal": _product("secondary_heating_removal", 270.0)}
+    )
+    repo = CompositeProductRepository(override=override, fallback=_ExplodingRepo())
+
+    # Act
+    product: Product = repo.get("secondary_heating_removal")
+
+    # Assert
+    assert product.measure_type == "secondary_heating_removal"
+    assert abs(product.unit_cost_per_m2 - 270.0) <= 1e-9
+
+
+def test_get_falls_through_to_the_fallback_when_the_override_misses() -> None:
+    # Arrange — the override has nothing for this type; the catalogue does.
+    override = _StaticRepo({})
+    fallback = _StaticRepo(
+        {"cavity_wall_insulation": _product("cavity_wall_insulation", 18.5)}
+    )
+    repo = CompositeProductRepository(override=override, fallback=fallback)
+
+    # Act
+    product: Product = repo.get("cavity_wall_insulation")
+
+    # Assert
+    assert product.measure_type == "cavity_wall_insulation"
+    assert abs(product.unit_cost_per_m2 - 18.5) <= 1e-9
+
+
+def test_get_raises_product_not_found_when_neither_source_has_it() -> None:
+    # Arrange
+    repo = CompositeProductRepository(
+        override=_StaticRepo({}), fallback=_StaticRepo({})
+    )
+
+    # Act / Assert
+    with pytest.raises(ProductNotFound):
+        repo.get("loft_insulation")
+
+
+def test_committed_override_file_prices_secondary_heating_removal() -> None:
+    # The committed overlay carries the flat per-dwelling decommission cost
+    # (ADR-0028, ported from the legacy heater_removal: (£25 + £200) × 1.2 VAT);
+    # contingency 0.25 is joined from config.
+    product: Product = ProductJsonRepository(OFF_CATALOGUE_COSTS_PATH).get(
+        "secondary_heating_removal"
+    )
+
+    assert abs(product.unit_cost_per_m2 - 270.0) <= 1e-9
+    assert abs(product.contingency_rate - 0.25) <= 1e-9