"""Tier-1 ratcheting Component Accuracy gate (ADR-0030). Runs the calculator-free leave-one-out scorer over the committed, anonymised fixture and asserts each per-component classification rate / geometry residual is no worse than a committed baseline. Because the prediction is deterministic and the fixture is frozen, every run reproduces the same numbers exactly — so a failure means a real *regression* in prediction quality, never sample noise. The floors / ceilings are the currently-measured values and only ever **tighten** (the repo's no-tolerance-widening ethos applied to an aggregate): when prediction improves, ratchet the relevant floor up in the same change. The end-to-end SAP / carbon / PE guards are deliberately *not* here — they need the calculator, whose API-path residual is a separate workstream; the component floors are the real gate (ADR-0030). """ from pathlib import Path import pytest from domain.epc_prediction.validation import ( ComponentAccuracy, evaluate_component_accuracy, ) from harness.epc_prediction_corpus import load_corpus _FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "epc_prediction" # Minimum classification hit-rate per component (ratchet floors). Tighten — never # loosen — as prediction improves. Values are the measured rates over the frozen # 36-target fixture; a 1e-3 tolerance absorbs float rounding only. # # Five floors were re-baselined when the per-cert-mapper-validation rework (#1245, # merged 2026-06-17) landed: that mapper re-derives both the predicted and the # *actual* EpcPropertyData the leave-one-out scorer compares, so its (Elmhurst- # validated) accuracy gains shifted the deterministic prediction agreement under # the prior floors. This is a ground-truth-method change, not a prediction-logic # loosening. The shifts are SAP-neutral: construction_age_band fell 0.6389->0.5000 # but every new miss is a single adjacent band (the ±1 `_pm1` floor below holds at # 0.8333) — the held-out actuals are unchanged; only the similarity-weighted donor # mode tipped, and it tipped entirely inside one near-tie pre-1900↔1900-29 (A↔B) # cohort. wall_insulation_type / floor_construction / has_hot_water_cylinder / has_pv # moved 3-6pp the same way. The tighten-only ratchet resumes from these new values. _RATE_FLOORS: dict[str, float] = { "wall_construction": 0.8889, "wall_insulation_type": 0.7778, "construction_age_band": 0.5000, "construction_age_band_pm1": 0.8333, "roof_construction": 0.7222, "floor_construction": 0.7812, "heating_main_fuel": 0.9722, "heating_main_category": 0.9444, "heating_main_control": 0.8056, "water_heating_fuel": 0.9722, "water_heating_code": 0.9444, "has_hot_water_cylinder": 0.8333, "cylinder_insulation_type": 0.5000, "secondary_heating_type": 0.0000, "roof_insulation_thickness": 0.4118, "roof_insulation_thickness_pm1": 0.4118, "floor_insulation": 0.9375, "has_room_in_roof": 0.8333, "modal_glazing_type": 0.5556, "has_pv": 0.9444, "solar_water_heating": 1.0000, } # Maximum mean absolute residual per numeric component (ratchet ceilings). # window_count is deliberately excluded — it is cosmetic for SAP (issue #1222): # the predicted picture clusters at a mapper-default 4 windows while actuals # spread 1-21, yet total_window_area (the SAP-relevant signal) stays tight. # # floor_area was re-baselined 11.8983 -> 12.0378 when floor-area sizing moved from # the plain cohort median to the geo-proximity-weighted median (a *method* change, # not a loosening). The change is a clear win on the full 514-target corpus # (MAE 10.48 -> 9.73 / MAPE 13.2% -> 12.2%); the n=36 frozen fixture moved +0.14 # the other way as small-sample noise (one target's shift moves an n=36 MAE more # than that). The ceiling still pins the new deterministic value exactly, so the # tighten-only ratchet resumes from here. _RESIDUAL_CEILINGS: dict[str, float] = { "floor_area": 12.0378, "total_window_area": 4.4067, "building_parts": 0.3333, "door_count": 0.6389, } _TOLERANCE = 1e-3 @pytest.fixture(scope="module") def accuracy() -> ComponentAccuracy: if not (_FIXTURE / "_index.json").exists(): pytest.skip(f"no EPC Prediction fixture at {_FIXTURE}") return evaluate_component_accuracy(load_corpus(_FIXTURE)) def test_fixture_yields_the_expected_target_count( accuracy: ComponentAccuracy, ) -> None: # The frozen fixture must still produce its full set of SAP-10.2 targets — a # drop means the fixture or the target filter changed. assert accuracy.targets >= 36 @pytest.mark.parametrize("component,floor", sorted(_RATE_FLOORS.items())) def test_classification_rate_does_not_regress( accuracy: ComponentAccuracy, component: str, floor: float ) -> None: # Arrange / Act rate = accuracy.rate(component) # Assert — the component is still applicable and at or above its floor. assert rate is not None, f"{component} had no applicable targets" assert rate >= floor - _TOLERANCE, ( f"{component} classification regressed: {rate:.4f} < floor {floor:.4f}" ) @pytest.mark.parametrize("component,ceiling", sorted(_RESIDUAL_CEILINGS.items())) def test_residual_does_not_regress( accuracy: ComponentAccuracy, component: str, ceiling: float ) -> None: # Arrange / Act mean_abs = accuracy.mean_abs_residual(component) # Assert — the mean absolute residual is at or below its ceiling. assert mean_abs is not None, f"{component} had no residuals" assert mean_abs <= ceiling + _TOLERANCE, ( f"{component} residual regressed: {mean_abs:.4f} > ceiling {ceiling:.4f}" )