Model/backend/documents_parser/tests/test_pashub_sap_accuracy.py

"""End-to-end accuracy test for the pashub site-notes extractor.

Chain under test (per fixture):

    pashub RdSAP site-note PDF
      -> parse_site_notes_pdf            (extractor + `from_site_notes` mapper)
      -> EpcPropertyData
      -> Sap10Calculator().calculate     -> SapResult.sap_score_continuous
      vs  pashub's own `pre_sap` rating  (from hubspot_deal_data)

Fixtures live in `fixtures/pashub_accuracy/` with a `manifest.json`; they are
built by `scripts/build_pashub_accuracy_fixtures.py` from the Guinness GMCA
project (see that script for provenance). `pre_sap` (e.g. `"C73"` -> 73) is
pashub's *own* RdSAP figure, so this gauge conflates two independent error
sources — our extractor+mapper AND our SAP-10 calculator (which alone lands
within 0.5 on only ~79% of the RdSAP corpus, see
`tests/infrastructure/epc_client/test_sap_accuracy_corpus.py`). It is therefore
a **hybrid** gate:

  * `test_pashub_fixture_computes`  — per fixture: the extractor must produce a
    calculable `EpcPropertyData`. Any *unexpected* exception is a hard fail (a
    real extractor/mapper bug on that property). The one *known* gap — the
    pashub mapper not resolving main heating `main_fuel_type` to a SAP fuel
    code (`MissingMainFuelType`) — is `xfail`ed dynamically, so these flip to
    passing automatically once that mapper fix lands.

  * `test_pashub_sap_accuracy_aggregate` — over the fixtures that compute:
    fraction within 0.5 SAP of `pre_sap`, and MAE. Ratcheting floor/ceiling
    (never loosen), mirroring the corpus gauge. While every fixture is blocked
    on the main-fuel-code gap the aggregate has no data and `xfail`s; once the
    mapper fix lands, ratchet the constants below to the observed values.
"""

from __future__ import annotations

import json
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Optional

import pytest

from backend.documents_parser.parser import parse_site_notes_pdf
from datatypes.epc.domain.mapper import UnmappedPasHubLabel
from domain.sap10_calculator.calculator import Sap10Calculator
from domain.sap10_calculator.exceptions import MissingMainFuelType, UnmappedSapCode

# Known in-progress pashub main-heating string→SAP-code mapper gaps. Some
# strict-raise at the mapper boundary during parse (`UnmappedPasHubLabel`, e.g.
# an unmapped "Bulk LPG" fuel), others at calculate time when the calculator
# meets a raw label it can't code (`UnmappedSapCode` for emitter / control,
# `MissingMainFuelType` for an unresolved fuel). Fixed field-by-field (mains-gas
# fuel code landed in #1554); each fix flips its fixtures from xfail to computing.
_KNOWN_GAPS: tuple[type[Exception], ...] = (
    MissingMainFuelType,
    UnmappedSapCode,
    UnmappedPasHubLabel,
)

_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy"
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"

# --- Ratchets (never loosen), mirroring test_sap_accuracy_corpus.py. ----------
# TODO(pashub-fuel-code): every fixture is currently blocked on the pashub
# main-fuel-code mapper gap, so the aggregate xfails and these bootstrap values
# are never asserted. Once that mapper fix lands and the aggregate first
# computes, ratchet these to the observed within-0.5 and MAE.
_MIN_WITHIN_HALF: float = 0.0
_MAX_SAP_MAE: float = 100.0

_KNOWN_GAP_REASON = (
    "pashub `from_site_notes` mapper does not int-code a main-heating string "
    "(fuel / emitter / control) to a SAP code; fix in progress field-by-field"
)


@dataclass(frozen=True)
class PashubFixture:
    deal_id: str
    uprn: Optional[int]
    pre_sap_raw: str
    pre_sap_score: int
    pdf: str

    @property
    def path(self) -> Path:
        return _FIXTURES_DIR / self.pdf


@dataclass(frozen=True)
class Outcome:
    """Result of running one fixture through the pipeline."""

    blocked: bool  # blocked on the known main-fuel-code gap (xfail territory)
    reason: Optional[str]
    sap_continuous: Optional[float]
    diff: Optional[float]  # abs(our SAP - pre_sap)


def _load_manifest() -> list[PashubFixture]:
    if not _MANIFEST_PATH.exists():
        return []
    data = json.loads(_MANIFEST_PATH.read_text())
    return [PashubFixture(**entry) for entry in data["fixtures"]]


_FIXTURES: list[PashubFixture] = _load_manifest()
_BY_ID: dict[str, PashubFixture] = {f.deal_id: f for f in _FIXTURES}
_IDS: list[str] = [f.deal_id for f in _FIXTURES]


@lru_cache(maxsize=None)
def _evaluate(deal_id: str) -> Outcome:
    """Run one fixture end-to-end. Shared across the two tests via the cache.

    Only `MissingMainFuelType` is swallowed (the known gap). Every other
    exception propagates — a real extractor/mapper bug the caller must surface.
    """
    fixture = _BY_ID[deal_id]
    try:
        epc = parse_site_notes_pdf(str(fixture.path), uprn=fixture.uprn)
        result = Sap10Calculator().calculate(epc)
    except _KNOWN_GAPS as exc:
        return Outcome(blocked=True, reason=str(exc), sap_continuous=None, diff=None)
    diff = abs(result.sap_score_continuous - fixture.pre_sap_score)
    return Outcome(
        blocked=False,
        reason=None,
        sap_continuous=result.sap_score_continuous,
        diff=diff,
    )


@pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy fixtures/manifest")
@pytest.mark.parametrize("deal_id", _IDS)
def test_pashub_fixture_computes(deal_id: str) -> None:
    """The extractor must yield a calculable dwelling for each real property."""
    outcome = _evaluate(deal_id)
    if outcome.blocked:
        pytest.xfail(f"{_KNOWN_GAP_REASON}: {outcome.reason}")
    assert outcome.sap_continuous is not None
    assert 1 <= outcome.sap_continuous <= 100


@pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy fixtures/manifest")
def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> None:
    """SAP within-0.5 of pashub's `pre_sap`, aggregated over computable fixtures."""
    diffs: list[float] = []
    blocked = 0
    errored = 0
    for deal_id in _IDS:
        try:
            outcome = _evaluate(deal_id)
        except Exception:  # noqa: BLE001 - a per-fixture bug; scored there, not here
            errored += 1
            continue
        if outcome.blocked:
            blocked += 1
            continue
        assert outcome.diff is not None
        diffs.append(outcome.diff)

    if not diffs:
        pytest.xfail(
            f"no fixture computes a SAP score "
            f"({blocked} blocked on the main-fuel-code gap, {errored} errored); "
            f"{_KNOWN_GAP_REASON}"
        )

    n = len(diffs)
    within_half = sum(1 for d in diffs if d < 0.5) / n
    sap_mae = sum(diffs) / n

    with capsys.disabled():
        print(
            f"\n[pashub Guinness | {n} computed / {blocked} blocked / "
            f"{errored} errored]"
            f"\n  SAP within-0.5 = {within_half:.1%}   MAE = {sap_mae:.3f}"
        )

    assert within_half >= _MIN_WITHIN_HALF, (
        f"SAP within-0.5 {within_half:.1%} fell below floor "
        f"{_MIN_WITHIN_HALF:.0%}"
    )
    assert sap_mae <= _MAX_SAP_MAE, (
        f"SAP MAE {sap_mae:.3f} exceeded ceiling {_MAX_SAP_MAE}"
    )