Model/backend/documents_parser/tests/test_pashub_sap_accuracy.py

"""End-to-end accuracy test for the pashub site-notes extractor.

Chain under test (per fixture):

    pashub RdSAP site-note PDF
      -> parse_site_notes_pdf            (extractor + `from_site_notes` mapper)
      -> EpcPropertyData
      -> Sap10Calculator().calculate     -> SapResult.sap_score_continuous
      vs  pashub's own `pre_sap` rating  (from hubspot_deal_data)

Fixtures live in `fixtures/pashub_accuracy/` with a `manifest.json`; they are
built by `scripts/build_pashub_accuracy_fixtures.py` from the Guinness GMCA
project (see that script for provenance). `pre_sap` (e.g. `"C73"` -> 73) is
pashub's *own* RdSAP figure, so this gauge conflates two independent error
sources — our extractor+mapper AND our SAP-10 calculator (which alone lands
within 0.5 on only ~79% of the RdSAP corpus, see
`tests/infrastructure/epc_client/test_sap_accuracy_corpus.py`). It is therefore
a **hybrid** gate:

  * `test_pashub_fixture_computes`  — per fixture: the extractor must produce a
    calculable `EpcPropertyData`. Any *unexpected* exception is a hard fail (a
    real extractor/mapper bug on that property). The one *known* remaining gap —
    the pashub mapper not coding a main-heating `main_heating_control` label to a
    SAP code — is `xfail`ed dynamically (see `_KNOWN_GAPS` for its two forms), so
    these flip to passing once that mapper fix lands. The main-fuel-code gap
    (`Bulk LPG`, and the `Fuel:`-form Mains Gas the extractor was dropping) was
    closed in #1558, so `MissingMainFuelType` is *no longer* swallowed — a
    fixture that regresses to a dropped fuel now hard-fails.

  * `test_pashub_sap_accuracy_aggregate` — over the fixtures that compute:
    fraction within 0.5 SAP of `pre_sap`, and MAE. Ratcheting floor/ceiling
    (never loosen), mirroring the corpus gauge. While every fixture is blocked
    on the main-heating-control gap the aggregate has no data and `xfail`s; once
    that mapper fix lands, ratchet the constants below to the observed values.
"""

from __future__ import annotations

import json
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Optional

import pytest

from backend.documents_parser.parser import parse_site_notes_pdf
from datatypes.epc.domain.mapper import UnmappedPasHubLabel
from domain.sap10_calculator.calculator import Sap10Calculator
from domain.sap10_calculator.exceptions import UnmappedSapCode

# Known in-progress pashub main-heating string→SAP-code mapper gaps, fixed
# field-by-field. The remaining gap is the `main_heating_control` label, which
# surfaces in one of two forms: `UnmappedSapCode` (raw label reaches the
# calculator) or, once the control mapper (#1557) lands, `UnmappedPasHubLabel`
# (strict-raised at the boundary for a control label not yet mapped — e.g. the
# cohort's community-heating charging control). Both xfail until that fix lands.
# The main-fuel-code gaps are closed (mains-gas code #1554; "Bulk LPG" + the
# `Fuel:`-form Mains Gas the extractor was dropping #1558), so `MissingMainFuelType`
# is deliberately *not* here — a fuel-drop regression hard-fails instead of hiding.
_KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel)

_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy"
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"

# --- Ratchets (never loosen at fixed coverage), mirroring test_sap_accuracy_corpus.py.
# The last integrated state computed 201/205 (4 blocked on residual
# `UnmappedPasHubLabel` gaps) at SAP within-0.5 = 12.4% (25/201), MAE = 2.560.
# Closing those last gaps — secondary fuel 'Mains Gas'/'House Coal' and the
# Table 4e Group 3 heat-network control 'Charging system linked to use of
# community heating, room thermostat only' — brings the cohort to **205/205
# computed (0 blocked)**. That is coverage growth, not a loosening: the
# within-0.5 *count* is unchanged at 25, so no fixture regressed; the fraction
# is 25/205 = 12.2% only because the 4 newly-computable fixtures all sit outside
# 0.5. MAE rises 2.560 -> 2.701 because two of those four are large, isolated
# divergences NOT caused by the mappings (which land the others at d=-1.7/+7.6):
#   - 499584755922  d=-33.0  Bulk-LPG main + house-coal secondary (34.0 vs pre_sap 67)
#   - 507644414148  d=+18.2  community heating (70.2 vs 52) at the time. Since
#                            RESOLVED: fuel/flags threaded (#1590 follow-up) and the
#                            control code corrected 2306->2308 (2026-07-15); now
#                            +2.0, the SAP-10.2-engine-vs-lodged offset (not a gap).
# Both are tracked for verification (pre_sap correctness vs a deeper modelling
# gap) — see docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md. Re-baselined to the
# full-coverage aggregate; ratchet back DOWN as those two are resolved.
# Ratcheted 0.12 -> 0.18 / 2.55 -> 1.75 by #1590 bug 4 (mechanical ventilation
# dispatched into the §2 ACH cascade instead of defaulting to NATURAL): 76/205
# cohort fixtures lodge a mechanical system; observed 18.5% within-0.5,
# MAE 1.730 — the campaign's largest single move.
# Ratcheted 0.18 -> 0.19 by the dig-round-3 fixes (see the MAE note below):
# observed 20.0% (41/205).
# Ratcheted 0.19 -> 0.23 by the window-orientation fix (see the MAE note
# below): observed 23.9% (49/205) — the campaign's largest within-0.5 move.
# Ratcheted 0.23 -> 0.50 by the #1596 extraction bundle (#1598 electric-shower
# + bath counts never threaded into `sap_heating`; #1599 floor label "Ground
# Floor" normalized to the calculator's "Ground floor" so the §5(12) suspended-
# timber infiltration term fires; #1600 water-heating "None" coded to the RdSAP
# §10.7 WHC 999 default instead of inheriting the gas combi). All three are
# silent-drop fixes the gov-API/Elmhurst paths already apply; verified against
# Khalim's ground truth (16 Stillwater 78.2 -> 75.7 vs 76; 58 Hackle 67.7 ->
# ~58 vs 59). Ratcheted 0.50 -> 0.51 by #1602 (semi-exposed / exposed floor
# exposure flags -> RdSAP §5.14 U=0.7 / §5.13 Table 20) + #1603 (alternative-
# wall sub-areas + extension floor construction now threaded): observed 51.2%
# (105/205). Ratcheted 0.51 -> 0.58 by the 2026-07-15 levers (low-energy-lights
# + has_draught_lobby threading in from_site_notes + heat-network control code
# 2306->2308): observed 58.5%.
# Ratcheted 0.58 -> 0.78 by the 2026-07-15c INFILTRATION bundle: from_site_notes
# uniquely dropped THREE §2 inputs the gov-API/Elmhurst mappers set —
# `percent_draughtproofed` (window §2(15) pinned at the 0.25-ACH worst case),
# the +0.25 m upper-storey joist void, and built-form `sheltered_sides`
# (defaulted to a flat 2, over-sheltering end/semi/detached). The three are
# NON-ADDITIVE — each overshoots alone (which is why #1601 upper-floor +0.25
# was previously RESOLVED as keep-raw: that call was confounded by the
# then-present draught-proofing drop suppressing every dwelling) — but together
# they land all 7 verified dwellings toward truth (none regress) and every
# built form near zero. Observed 79.5% here; vs the DB SAP-10.2 oracle
# (property_baseline_performance.effective_sap_score, portfolio 838) the cohort
# moves 62.1% -> 83.3% within-0.5. gov-API RdSAP corpus unchanged (78.9%).
_MIN_WITHIN_HALF: float = 0.78
# Ratcheted 2.71 -> 2.64 by #1590 bug 2 (roof "Insulation At: None" -> explicit
# zero thickness): observed MAE 2.632 — the two ~7-SAP roof over-raters
# (507665533138 / 507670893756) correcting is exactly 14/205 ≈ 0.07 of MAE.
# Ratcheted 2.64 -> 2.58 by #1590 bug 1 (surveyed PV arrays now reach the
# Appendix M input): observed MAE 2.571 — 507639151843's 13.5-SAP zero-PV-credit
# under-rate closing.
# Ratcheted 2.58 -> 2.55 by #1590 bug 3 (PV Connection label -> gov-API int:
# "Not connected" no longer silently credited): observed MAE 2.538.
# Ratcheted 1.75 -> 1.70 by the dig-round-2 fixes: RIR surfaces billed with
# the surveyed roof insulation + stud-wall commons, and community heating
# lodging Table 4a code 301 + Table 12 fuel 51 (507644414148 computes on the
# heat-network branch). Observed MAE 1.684.
# Ratcheted 1.70 -> 1.56 after Khalim manually verified 4 suspect pre_sap
# values in pashub (2026-07-14): 9 Council Houses CW5 8AP is F33 (manifest had
# a stale D67 — our 34.0 was right within a point) and Brightholme M11 4WE is
# the accredited 43 (was pashub's preliminary 53). 58 Hackle Street D58 and
# 16 Bingley Close E52 were CONFIRMED correct — our +7.8/+5.7 deviations there
# are extraction bugs to hunt. Observed MAE 1.551.
# Ratcheted 1.56 -> 1.39 (and within-0.5 0.18 -> 0.19) by the dig-round-3
# fixes: (a) RIR roof-insulation threading now respects the per-surface
# "insulation known? No" answers (Brightholme, whose loft hatch was screwed
# shut, +7.3 -> +0.6); (b) the surveyed "Wall Dry-Lined? Yes" (39/205
# fixtures) reaches the §5.8/Table 14 wall-U adjustment; (c) the Bingley
# community residual — cylinder insulation label int-coded so storage loss
# bills, Table 3 primary loss + RdSAP p.59 flat-rate charging default on the
# water-only heat-network branch (+5.7 -> +3.2). Observed 20.0% / MAE 1.383.
# Held at 1.39 (not loosened) after the secondary-heating-type coding fix
# (raw "Secondary System:" label int-coded to its Table 4a room-heater code
# instead of silently defaulting to code 693 eff 1.00): correctness fix,
# 499607224524 +4.0 -> +1.2, observed MAE 1.383 -> 1.381 — too small to
# tighten the 2-decimal ceiling, within-0.5 unchanged at 20.0%.
# Ratcheted 1.39 -> 1.37 by the Simplified-RIR common-wall fix: a "Roof room
# type" (RdSAP §3.9.2) lodgement now bills common walls as `common_wall` at
# the main-wall U (spec area L x (0.25 + H)) instead of a Detailed `stud_wall`
# roof surface that dropped to the Table 18 col(4) 2.30 uninsulated default
# (78/72 North Road, +1.1 SAP each). Observed MAE 1.3703, within-0.5 20.0%.
# DELIBERATELY LOOSENED 1.371 -> 1.41 by the window-orientation fix (Khalim
# authorised, 2026-07-15) — the ONE sanctioned exception to "ratchets never
# loosen". PasHub lodged `orientation` as a raw string ("South East") where
# the solar-gains cascade only accepts the int octant 1..8, so EVERY window's
# solar gain was silently zeroed (100% of the cohort, ~1,647 windows). Coding
# it (`_pashub_orientation_int`) is an unambiguous correctness fix that moves
# within-0.5 20.0% -> 23.9% (41 -> 49; the campaign's largest within-0.5 move)
# and within-1 87 -> 98. MAE RISES 1.370 -> 1.405 ONLY because the zeroed
# gains were masking the known SAP-10.2-engine-vs-lodged-2012-RdSAP offset the
# module docstring already flags (bias -0.67 -> +1.04) — NOT an extraction
# regression. The correct oracle for this field is the Elmhurst 10.2 worksheet,
# not the hybrid `pre_sap` gauge; validate there before tightening further.
# Nudged 1.41 -> 1.411 by the #1592 revert of the water-only heat-network
# CALCULATOR changes (primary loss + flat-rate charging leaked into the gov-API
# RdSAP corpus). 16 Bingley Close reverts +3.2 -> +4.3, so observed MAE 1.405 ->
# 1.4103; re-tighten when the water-only branch is fixed correctly under #1592
# (the cylinder-insulation MAPPER fix is retained, so the revert is partial).
# DELIBERATELY LOOSENED 1.411 -> 1.415 by the #1592 sub-task B water-only DHW
# fix (16 Bingley Close, WHC 950, +5.946 -> +6.818 = +0.0043 MAE) — a sanctioned
# "never loosen" exception that expected re-tightening here.
# RE-TIGHTENED 1.415 -> 0.653 by the #1596 extraction bundle (#1598 electric-
# shower + bath counts; #1599 floor label "Ground Floor" -> "Ground floor" so the
# §5(12) suspended-timber infiltration fires; #1600 water-heating "None" -> RdSAP
# §10.7 WHC 999 — see the within-0.5 note above), which SUPERSEDES the #1592
# loosening: MAE 0.6517 on the merged tree (with the water-only DHW fix, 16
# Bingley still +6.818). The three silent-drop fixes the gov-API/Elmhurst paths
# already apply; the electric-shower drop (100/205) was the dominant lever.
# Ratcheted 0.653 -> 0.625 by #1602 (floor exposure flags) + #1603 (alternative
# walls + extension floor construction): observed MAE 0.624.
# Ratcheted 0.625 -> 0.521 by the 2026-07-15 levers (low-energy-lights +
# has_draught_lobby + heat-network control 2306->2308): observed MAE 0.520.
# Ratcheted 0.521 -> 0.40 by the 2026-07-15c infiltration bundle
# (percent_draughtproofed + upper-floor +0.25 m + built-form sheltered_sides;
# #1601 upper-floor RE-ADJUDICATED, see the within-0.5 note above): observed
# MAE 0.389.
_MAX_SAP_MAE: float = 0.40

_KNOWN_GAP_REASON = (
    "pashub `from_site_notes` mapper does not int-code a main-heating "
    "`main_heating_control` label to a SAP code; fix in progress field-by-field"
)


@dataclass(frozen=True)
class PashubFixture:
    deal_id: str
    uprn: Optional[int]
    pre_sap_raw: str
    pre_sap_score: int
    pdf: str

    @property
    def path(self) -> Path:
        return _FIXTURES_DIR / self.pdf


@dataclass(frozen=True)
class Outcome:
    """Result of running one fixture through the pipeline."""

    blocked: bool  # blocked on the known main-heating-control gap (xfail territory)
    reason: Optional[str]
    sap_continuous: Optional[float]
    diff: Optional[float]  # abs(our SAP - pre_sap)


def _load_manifest() -> list[PashubFixture]:
    if not _MANIFEST_PATH.exists():
        return []
    data = json.loads(_MANIFEST_PATH.read_text())
    return [PashubFixture(**entry) for entry in data["fixtures"]]


_FIXTURES: list[PashubFixture] = _load_manifest()
_BY_ID: dict[str, PashubFixture] = {f.deal_id: f for f in _FIXTURES}
_IDS: list[str] = [f.deal_id for f in _FIXTURES]


@lru_cache(maxsize=None)
def _evaluate(deal_id: str) -> Outcome:
    """Run one fixture end-to-end. Shared across the two tests via the cache.

    Only `UnmappedSapCode` is swallowed (the known control-label gap). Every
    other exception propagates — a real extractor/mapper bug the caller must
    surface.
    """
    fixture = _BY_ID[deal_id]
    try:
        epc = parse_site_notes_pdf(str(fixture.path), uprn=fixture.uprn)
        result = Sap10Calculator().calculate(epc)
    except _KNOWN_GAPS as exc:
        return Outcome(blocked=True, reason=str(exc), sap_continuous=None, diff=None)
    diff = abs(result.sap_score_continuous - fixture.pre_sap_score)
    return Outcome(
        blocked=False,
        reason=None,
        sap_continuous=result.sap_score_continuous,
        diff=diff,
    )


@pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy fixtures/manifest")
@pytest.mark.parametrize("deal_id", _IDS)
def test_pashub_fixture_computes(deal_id: str) -> None:
    """The extractor must yield a calculable dwelling for each real property."""
    outcome = _evaluate(deal_id)
    if outcome.blocked:
        pytest.xfail(f"{_KNOWN_GAP_REASON}: {outcome.reason}")
    assert outcome.sap_continuous is not None
    assert 1 <= outcome.sap_continuous <= 100


@pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy fixtures/manifest")
def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> None:
    """SAP within-0.5 of pashub's `pre_sap`, aggregated over computable fixtures."""
    diffs: list[float] = []
    blocked = 0
    errored = 0
    for deal_id in _IDS:
        try:
            outcome = _evaluate(deal_id)
        except Exception:  # noqa: BLE001 - a per-fixture bug; scored there, not here
            errored += 1
            continue
        if outcome.blocked:
            blocked += 1
            continue
        assert outcome.diff is not None
        diffs.append(outcome.diff)

    if not diffs:
        pytest.xfail(
            f"no fixture computes a SAP score "
            f"({blocked} blocked on the main-heating-control gap, {errored} errored); "
            f"{_KNOWN_GAP_REASON}"
        )

    n = len(diffs)
    within_half = sum(1 for d in diffs if d < 0.5) / n
    sap_mae = sum(diffs) / n

    with capsys.disabled():
        print(
            f"\n[pashub Guinness | {n} computed / {blocked} blocked / "
            f"{errored} errored]"
            f"\n  SAP within-0.5 = {within_half:.1%}   MAE = {sap_mae:.3f}"
        )

    assert within_half >= _MIN_WITHIN_HALF, (
        f"SAP within-0.5 {within_half:.1%} fell below floor "
        f"{_MIN_WITHIN_HALF:.0%}"
    )
    assert sap_mae <= _MAX_SAP_MAE, (
        f"SAP MAE {sap_mae:.3f} exceeded ceiling {_MAX_SAP_MAE}"
    )