Stop swallowing closed PAS Hub main-fuel gaps in the accuracy harness 🟪

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Roth 2026-07-14 17:24:15 +00:00
parent 8529c71ff2
commit 88f221b86e

View file

@ -19,16 +19,19 @@ a **hybrid** gate:
* `test_pashub_fixture_computes` per fixture: the extractor must produce a
calculable `EpcPropertyData`. Any *unexpected* exception is a hard fail (a
real extractor/mapper bug on that property). The one *known* gap the
pashub mapper not resolving main heating `main_fuel_type` to a SAP fuel
code (`MissingMainFuelType`) is `xfail`ed dynamically, so these flip to
passing automatically once that mapper fix lands.
real extractor/mapper bug on that property). The one *known* remaining gap
the pashub mapper not coding a main-heating `main_heating_control` label to a
SAP code (`UnmappedSapCode`) is `xfail`ed dynamically, so these flip to
passing automatically once that mapper fix lands. The main-fuel-code gap
(`Bulk LPG`, and the `Fuel:`-form Mains Gas the extractor was dropping) was
closed in #1558, so those exceptions are *no longer* swallowed — a fixture
that regresses to a fuel gap now hard-fails.
* `test_pashub_sap_accuracy_aggregate` over the fixtures that compute:
fraction within 0.5 SAP of `pre_sap`, and MAE. Ratcheting floor/ceiling
(never loosen), mirroring the corpus gauge. While every fixture is blocked
on the main-fuel-code gap the aggregate has no data and `xfail`s; once the
mapper fix lands, ratchet the constants below to the observed values.
on the main-heating-control gap the aggregate has no data and `xfail`s; once
that mapper fix lands, ratchet the constants below to the observed values.
"""
from __future__ import annotations
@ -42,36 +45,32 @@ from typing import Optional
import pytest
from backend.documents_parser.parser import parse_site_notes_pdf
from datatypes.epc.domain.mapper import UnmappedPasHubLabel
from domain.sap10_calculator.calculator import Sap10Calculator
from domain.sap10_calculator.exceptions import MissingMainFuelType, UnmappedSapCode
from domain.sap10_calculator.exceptions import UnmappedSapCode
# Known in-progress pashub main-heating string→SAP-code mapper gaps. Some
# strict-raise at the mapper boundary during parse (`UnmappedPasHubLabel`, e.g.
# an unmapped "Bulk LPG" fuel), others at calculate time when the calculator
# meets a raw label it can't code (`UnmappedSapCode` for emitter / control,
# `MissingMainFuelType` for an unresolved fuel). Fixed field-by-field (mains-gas
# fuel code landed in #1554); each fix flips its fixtures from xfail to computing.
_KNOWN_GAPS: tuple[type[Exception], ...] = (
MissingMainFuelType,
UnmappedSapCode,
UnmappedPasHubLabel,
)
# Known in-progress pashub main-heating string→SAP-code mapper gaps, fixed
# field-by-field. The remaining gap is a raw `main_heating_control` label the
# calculator can't code (`UnmappedSapCode`) — it xfails until that mapper fix
# lands. The main-fuel-code gaps are closed: mains-gas code landed in #1554, and
# #1558 coded "Bulk LPG" and captured the `Fuel:`-form Mains Gas the extractor
# was dropping. Their exceptions (`UnmappedPasHubLabel`, `MissingMainFuelType`)
# are deliberately *not* here, so a fuel regression hard-fails instead of hiding.
_KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode,)
_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy"
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"
# --- Ratchets (never loosen), mirroring test_sap_accuracy_corpus.py. ----------
# TODO(pashub-fuel-code): every fixture is currently blocked on the pashub
# main-fuel-code mapper gap, so the aggregate xfails and these bootstrap values
# are never asserted. Once that mapper fix lands and the aggregate first
# TODO(pashub-control-code): every fixture is currently blocked on the pashub
# main-heating-control mapper gap, so the aggregate xfails and these bootstrap
# values are never asserted. Once that mapper fix lands and the aggregate first
# computes, ratchet these to the observed within-0.5 and MAE.
_MIN_WITHIN_HALF: float = 0.0
_MAX_SAP_MAE: float = 100.0
_KNOWN_GAP_REASON = (
"pashub `from_site_notes` mapper does not int-code a main-heating string "
"(fuel / emitter / control) to a SAP code; fix in progress field-by-field"
"pashub `from_site_notes` mapper does not int-code a main-heating "
"`main_heating_control` label to a SAP code; fix in progress field-by-field"
)
@ -92,7 +91,7 @@ class PashubFixture:
class Outcome:
"""Result of running one fixture through the pipeline."""
blocked: bool # blocked on the known main-fuel-code gap (xfail territory)
blocked: bool # blocked on the known main-heating-control gap (xfail territory)
reason: Optional[str]
sap_continuous: Optional[float]
diff: Optional[float] # abs(our SAP - pre_sap)
@ -114,8 +113,9 @@ _IDS: list[str] = [f.deal_id for f in _FIXTURES]
def _evaluate(deal_id: str) -> Outcome:
"""Run one fixture end-to-end. Shared across the two tests via the cache.
Only `MissingMainFuelType` is swallowed (the known gap). Every other
exception propagates a real extractor/mapper bug the caller must surface.
Only `UnmappedSapCode` is swallowed (the known control-label gap). Every
other exception propagates a real extractor/mapper bug the caller must
surface.
"""
fixture = _BY_ID[deal_id]
try:
@ -164,7 +164,7 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No
if not diffs:
pytest.xfail(
f"no fixture computes a SAP score "
f"({blocked} blocked on the main-fuel-code gap, {errored} errored); "
f"({blocked} blocked on the main-heating-control gap, {errored} errored); "
f"{_KNOWN_GAP_REASON}"
)