Model/tests/domain/epc_prediction/test_component_accuracy_gate.py

"""Tier-1 ratcheting Component Accuracy gate (ADR-0030).

Runs the calculator-free leave-one-out scorer over the committed, anonymised
fixture and asserts each per-component classification rate / geometry residual is
no worse than a committed baseline. Because the prediction is deterministic and
the fixture is frozen, every run reproduces the same numbers exactly — so a
failure means a real *regression* in prediction quality, never sample noise.

The floors / ceilings are the currently-measured values and only ever **tighten**
(the repo's no-tolerance-widening ethos applied to an aggregate): when prediction
improves, ratchet the relevant floor up in the same change. The end-to-end
SAP / carbon / PE guards are deliberately *not* here — they need the calculator,
whose API-path residual is a separate workstream; the component floors are the
real gate (ADR-0030).
"""

from pathlib import Path

import pytest

from domain.epc_prediction.validation import (
    ComponentAccuracy,
    evaluate_component_accuracy,
)
from harness.epc_prediction_corpus import load_corpus

_FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "epc_prediction"

# Minimum classification hit-rate per component (ratchet floors). Tighten — never
# loosen — as prediction improves. Values are the measured rates over the frozen
# 36-target fixture; a 1e-3 tolerance absorbs float rounding only.
#
# Five floors were re-baselined when the per-cert-mapper-validation rework (#1245,
# merged 2026-06-17) landed: that mapper re-derives both the predicted and the
# *actual* EpcPropertyData the leave-one-out scorer compares, so its (Elmhurst-
# validated) accuracy gains shifted the deterministic prediction agreement under
# the prior floors. This is a ground-truth-method change, not a prediction-logic
# loosening. The shifts are SAP-neutral: construction_age_band fell 0.6389->0.5000
# but every new miss is a single adjacent band (the ±1 `_pm1` floor below holds at
# 0.8333) — the held-out actuals are unchanged; only the similarity-weighted donor
# mode tipped, and it tipped entirely inside one near-tie pre-1900↔1900-29 (A↔B)
# cohort. wall_insulation_type / floor_construction / has_hot_water_cylinder / has_pv
# moved 3-6pp the same way. The tighten-only ratchet resumes from these new values.
#
# Re-baselined again under ADR-0037 (full-SAP mapper completion): full-SAP
# (on-construction) certs previously mapped property_type=None, so the hard cohort
# filter (comparable_properties.py — `c.epc.property_type == target.property_type`)
# silently excluded them from EVERY cohort, as donors and as targets. Mapping
# property_type correctly admits these real lodged EPCs as comparables — another
# ground-truth-method change. Net effect over the n=36 fixture: **16 components
# better, 4 worse, 6 unchanged**. The gains are concentrated in the physical /
# geometric characteristics full-SAP certs measure accurately — window_count
# residual 3.83->1.69, total_window_area 3.82->3.72, building_parts 0.33->0.12,
# floor_construction 0.78->0.91, construction_age_band 0.50->0.78, modal_glazing
# 0.56->0.84, walls/room-in-roof/heating-control all up. The 4 that fell are the
# new-build-vs-old-stock service mismatch on 1-2 targets each (heating_main_fuel
# 0.9722->0.9394, water_heating_fuel ->0.9495, cylinder_insulation_type 0.6667->
# 0.3333) plus floor_area (+0.31 MAE). Tighten-only resumes from these values.
#
# has_pv re-baselined 0.9798->0.9697 when full-SAP lodged PV mapping landed
# (datatypes/epc/domain/mapper.py `_sap_17_1_pv_arrays`): full-SAP certs lodge
# their measured array under `sap_energy_source.pv_arrays`, which the schema
# dropped at parse, so the leave-one-out scorer's *actual* has_pv read False for
# every full-SAP PV dwelling. Carrying the array now reads the true has_pv=True,
# and one full-SAP target the similarity-weighted donors don't predict as PV
# tips the agreement 32/33 (the held-out actual is now correct — a ground-truth-
# method change, not a prediction-logic loosening). Tighten-only resumes here.
#
# has_hot_water_cylinder 0.8687->0.8586, cylinder_insulation_type 0.3333->0.1667,
# door_count residual 0.3131->0.3333 re-baselined when the mapper started
# deriving `multiple_glazed_proportion` (single->0 / double->100) from an
# unambiguous window.description for 16.x certs whose "ND" multiple_glazing_type
# previously left the numeric field entirely absent — RdSapSchema17_1 requires
# it, so the cert failed loud and was excluded from the donor pool wholesale
# (worklist P4; see the mapper comment at `_normalize_sap_schema_16_x`).
# Motivating case: cert 0141-2860-6891-9124-5625 / uprn 100021969385 (property
# 753950), lodging "Fully double glazed" with the proportion field missing —
# previously hard-failed the whole modelling_e2e batch it was in. Admitting
# these certs is a ground-truth-method change (more real lodged EPCs enter the
# comparable pool), not a prediction-logic loosening; the tip is donor-pool
# composition on a handful of near-tie matches, same mechanism as the #1245 /
# ADR-0037 re-baselines above. A flat `multiple_glazed_proportion=100` default
# was tried earlier and reverted for the same reason without a derivation
# rationale; deriving it from explicit "single"/"double" wording is the
# targeted fix worklist P4 called for. Tighten-only resumes from these values.
_RATE_FLOORS: dict[str, float] = {
    "wall_construction": 0.9091,
    "wall_insulation_type": 0.8687,
    "construction_age_band": 0.7778,
    "construction_age_band_pm1": 0.9091,
    "roof_construction": 0.7222,
    "floor_construction": 0.9053,
    "heating_main_fuel": 0.9394,
    "heating_main_category": 0.9596,
    "heating_main_control": 0.9091,
    "water_heating_fuel": 0.9495,
    "water_heating_code": 0.9798,
    "has_hot_water_cylinder": 0.8586,
    "cylinder_insulation_type": 0.1667,
    "secondary_heating_type": 0.0000,
    "roof_insulation_thickness": 0.4118,
    "roof_insulation_thickness_pm1": 0.4118,
    "floor_insulation": 0.9375,
    "has_room_in_roof": 0.9495,
    "modal_glazing_type": 0.8384,
    "has_pv": 0.9697,
    "solar_water_heating": 1.0000,
}

# Maximum mean absolute residual per numeric component (ratchet ceilings).
# window_count is deliberately excluded — it is cosmetic for SAP (issue #1222):
# the predicted picture clusters at a mapper-default 4 windows while actuals
# spread 1-21, yet total_window_area (the SAP-relevant signal) stays tight.
#
# floor_area was re-baselined 11.8983 -> 12.0378 when floor-area sizing moved from
# the plain cohort median to the geo-proximity-weighted median (a *method* change,
# not a loosening). The change is a clear win on the full 514-target corpus
# (MAE 10.48 -> 9.73 / MAPE 13.2% -> 12.2%); the n=36 frozen fixture moved +0.14
# the other way as small-sample noise (one target's shift moves an n=36 MAE more
# than that). The ceiling still pins the new deterministic value exactly, so the
# tighten-only ratchet resumes from here.
# total_window_area / building_parts / door_count all tightened under ADR-0037
# (full-SAP certs admitted as donors — their measured geometry sharpens the
# dimensional predictions); floor_area loosened 12.0378 -> 12.0586 as the one
# physical residual that fell (1-2 targets picking a new-build donor). See the
# _RATE_FLOORS note above.
_RESIDUAL_CEILINGS: dict[str, float] = {
    "floor_area": 12.0586,
    "total_window_area": 3.7184,
    "building_parts": 0.1212,
    "door_count": 0.3333,
}

_TOLERANCE = 1e-3


@pytest.fixture(scope="module")
def accuracy() -> ComponentAccuracy:
    if not (_FIXTURE / "_index.json").exists():
        pytest.skip(f"no EPC Prediction fixture at {_FIXTURE}")
    return evaluate_component_accuracy(load_corpus(_FIXTURE))


def test_fixture_yields_the_expected_target_count(
    accuracy: ComponentAccuracy,
) -> None:
    # The frozen fixture must still produce its full set of SAP-10.2 targets — a
    # drop means the fixture or the target filter changed.
    assert accuracy.targets >= 36


@pytest.mark.parametrize("component,floor", sorted(_RATE_FLOORS.items()))
def test_classification_rate_does_not_regress(
    accuracy: ComponentAccuracy, component: str, floor: float
) -> None:
    # Arrange / Act
    rate = accuracy.rate(component)

    # Assert — the component is still applicable and at or above its floor.
    assert rate is not None, f"{component} had no applicable targets"
    assert rate >= floor - _TOLERANCE, (
        f"{component} classification regressed: {rate:.4f} < floor {floor:.4f}"
    )


@pytest.mark.parametrize("component,ceiling", sorted(_RESIDUAL_CEILINGS.items()))
def test_residual_does_not_regress(
    accuracy: ComponentAccuracy, component: str, ceiling: float
) -> None:
    # Arrange / Act
    mean_abs = accuracy.mean_abs_residual(component)

    # Assert — the mean absolute residual is at or below its ceiling.
    assert mean_abs is not None, f"{component} had no residuals"
    assert mean_abs <= ceiling + _TOLERANCE, (
        f"{component} residual regressed: {mean_abs:.4f} > ceiling {ceiling:.4f}"
    )