mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-27 23:35:01 +00:00
Stop swallowing closed PAS Hub main-fuel gaps in the accuracy harness 🟪
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8529c71ff2
commit
88f221b86e
1 changed files with 28 additions and 28 deletions
|
|
@ -19,16 +19,19 @@ a **hybrid** gate:
|
|||
|
||||
* `test_pashub_fixture_computes` — per fixture: the extractor must produce a
|
||||
calculable `EpcPropertyData`. Any *unexpected* exception is a hard fail (a
|
||||
real extractor/mapper bug on that property). The one *known* gap — the
|
||||
pashub mapper not resolving main heating `main_fuel_type` to a SAP fuel
|
||||
code (`MissingMainFuelType`) — is `xfail`ed dynamically, so these flip to
|
||||
passing automatically once that mapper fix lands.
|
||||
real extractor/mapper bug on that property). The one *known* remaining gap —
|
||||
the pashub mapper not coding a main-heating `main_heating_control` label to a
|
||||
SAP code (`UnmappedSapCode`) — is `xfail`ed dynamically, so these flip to
|
||||
passing automatically once that mapper fix lands. The main-fuel-code gap
|
||||
(`Bulk LPG`, and the `Fuel:`-form Mains Gas the extractor was dropping) was
|
||||
closed in #1558, so those exceptions are *no longer* swallowed — a fixture
|
||||
that regresses to a fuel gap now hard-fails.
|
||||
|
||||
* `test_pashub_sap_accuracy_aggregate` — over the fixtures that compute:
|
||||
fraction within 0.5 SAP of `pre_sap`, and MAE. Ratcheting floor/ceiling
|
||||
(never loosen), mirroring the corpus gauge. While every fixture is blocked
|
||||
on the main-fuel-code gap the aggregate has no data and `xfail`s; once the
|
||||
mapper fix lands, ratchet the constants below to the observed values.
|
||||
on the main-heating-control gap the aggregate has no data and `xfail`s; once
|
||||
that mapper fix lands, ratchet the constants below to the observed values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -42,36 +45,32 @@ from typing import Optional
|
|||
import pytest
|
||||
|
||||
from backend.documents_parser.parser import parse_site_notes_pdf
|
||||
from datatypes.epc.domain.mapper import UnmappedPasHubLabel
|
||||
from domain.sap10_calculator.calculator import Sap10Calculator
|
||||
from domain.sap10_calculator.exceptions import MissingMainFuelType, UnmappedSapCode
|
||||
from domain.sap10_calculator.exceptions import UnmappedSapCode
|
||||
|
||||
# Known in-progress pashub main-heating string→SAP-code mapper gaps. Some
|
||||
# strict-raise at the mapper boundary during parse (`UnmappedPasHubLabel`, e.g.
|
||||
# an unmapped "Bulk LPG" fuel), others at calculate time when the calculator
|
||||
# meets a raw label it can't code (`UnmappedSapCode` for emitter / control,
|
||||
# `MissingMainFuelType` for an unresolved fuel). Fixed field-by-field (mains-gas
|
||||
# fuel code landed in #1554); each fix flips its fixtures from xfail to computing.
|
||||
_KNOWN_GAPS: tuple[type[Exception], ...] = (
|
||||
MissingMainFuelType,
|
||||
UnmappedSapCode,
|
||||
UnmappedPasHubLabel,
|
||||
)
|
||||
# Known in-progress pashub main-heating string→SAP-code mapper gaps, fixed
|
||||
# field-by-field. The remaining gap is a raw `main_heating_control` label the
|
||||
# calculator can't code (`UnmappedSapCode`) — it xfails until that mapper fix
|
||||
# lands. The main-fuel-code gaps are closed: mains-gas code landed in #1554, and
|
||||
# #1558 coded "Bulk LPG" and captured the `Fuel:`-form Mains Gas the extractor
|
||||
# was dropping. Their exceptions (`UnmappedPasHubLabel`, `MissingMainFuelType`)
|
||||
# are deliberately *not* here, so a fuel regression hard-fails instead of hiding.
|
||||
_KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode,)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy"
|
||||
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"
|
||||
|
||||
# --- Ratchets (never loosen), mirroring test_sap_accuracy_corpus.py. ----------
|
||||
# TODO(pashub-fuel-code): every fixture is currently blocked on the pashub
|
||||
# main-fuel-code mapper gap, so the aggregate xfails and these bootstrap values
|
||||
# are never asserted. Once that mapper fix lands and the aggregate first
|
||||
# TODO(pashub-control-code): every fixture is currently blocked on the pashub
|
||||
# main-heating-control mapper gap, so the aggregate xfails and these bootstrap
|
||||
# values are never asserted. Once that mapper fix lands and the aggregate first
|
||||
# computes, ratchet these to the observed within-0.5 and MAE.
|
||||
_MIN_WITHIN_HALF: float = 0.0
|
||||
_MAX_SAP_MAE: float = 100.0
|
||||
|
||||
_KNOWN_GAP_REASON = (
|
||||
"pashub `from_site_notes` mapper does not int-code a main-heating string "
|
||||
"(fuel / emitter / control) to a SAP code; fix in progress field-by-field"
|
||||
"pashub `from_site_notes` mapper does not int-code a main-heating "
|
||||
"`main_heating_control` label to a SAP code; fix in progress field-by-field"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -92,7 +91,7 @@ class PashubFixture:
|
|||
class Outcome:
|
||||
"""Result of running one fixture through the pipeline."""
|
||||
|
||||
blocked: bool # blocked on the known main-fuel-code gap (xfail territory)
|
||||
blocked: bool # blocked on the known main-heating-control gap (xfail territory)
|
||||
reason: Optional[str]
|
||||
sap_continuous: Optional[float]
|
||||
diff: Optional[float] # abs(our SAP - pre_sap)
|
||||
|
|
@ -114,8 +113,9 @@ _IDS: list[str] = [f.deal_id for f in _FIXTURES]
|
|||
def _evaluate(deal_id: str) -> Outcome:
|
||||
"""Run one fixture end-to-end. Shared across the two tests via the cache.
|
||||
|
||||
Only `MissingMainFuelType` is swallowed (the known gap). Every other
|
||||
exception propagates — a real extractor/mapper bug the caller must surface.
|
||||
Only `UnmappedSapCode` is swallowed (the known control-label gap). Every
|
||||
other exception propagates — a real extractor/mapper bug the caller must
|
||||
surface.
|
||||
"""
|
||||
fixture = _BY_ID[deal_id]
|
||||
try:
|
||||
|
|
@ -164,7 +164,7 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No
|
|||
if not diffs:
|
||||
pytest.xfail(
|
||||
f"no fixture computes a SAP score "
|
||||
f"({blocked} blocked on the main-fuel-code gap, {errored} errored); "
|
||||
f"({blocked} blocked on the main-heating-control gap, {errored} errored); "
|
||||
f"{_KNOWN_GAP_REASON}"
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue