diff --git a/backend/documents_parser/extractor.py b/backend/documents_parser/extractor.py index 47022c551..fe85c8f59 100644 --- a/backend/documents_parser/extractor.py +++ b/backend/documents_parser/extractor.py @@ -563,7 +563,9 @@ class PasHubRdSapSiteNotesExtractor: manufacturer=self._get_in(data, "Manufacturer") or "", model=self._get_in(data, "Model") or "", orig_manufacturer=self._get_in(data, "Orig Manuf") or "", - fuel=self._get_in(data, "Fuel") or "", + fuel=self._get_in(data, "Fuel") + or self._get_in(data, "Fuel:") + or "", summer_efficiency=float(self._get_in(data, "S. Efficiency") or 0), type=self._get_in(data, "Type") or "", condensing=self._bool_in(data, "Condensing"), diff --git a/backend/documents_parser/tests/test_extractor.py b/backend/documents_parser/tests/test_extractor.py index be577f1bf..438c8a74d 100644 --- a/backend/documents_parser/tests/test_extractor.py +++ b/backend/documents_parser/tests/test_extractor.py @@ -537,6 +537,44 @@ class TestHeatingAndHotWater: ) +class TestMainHeatingFuelLabelVariant: + """The newer PAS Hub site-note form lodges the main-heating fuel under a + `Fuel:` label (with colon) rather than the usual `Fuel`. The extractor must + recognise both, else the fuel is silently dropped and the property presents + as fuel-less downstream — the 3 Guinness cohort "blank-fuel" residuals that + were really Mains Gas all along (issue #1558).""" + + def _section(self, fuel_label: str) -> list[str]: + return [ + "Heating & Hot Water", + "System type:", + "Boiler with radiators or underfloor heating", + fuel_label, + "Mains Gas", + "Ventilation", + ] + + def test_colon_fuel_label_captures_value(self) -> None: + # Arrange + tokens = self._section("Fuel:") + + # Act + hhw = PasHubRdSapSiteNotesExtractor(tokens).extract_heating_and_hot_water() + + # Assert + assert hhw.main_heating.fuel == "Mains Gas" + + def test_plain_fuel_label_still_captures_value(self) -> None: + # Arrange + tokens = self._section("Fuel") + + # Act + hhw = PasHubRdSapSiteNotesExtractor(tokens).extract_heating_and_hot_water() + + # Assert + assert hhw.main_heating.fuel == "Mains Gas" + + class TestVentilation: @pytest.fixture def ventilation(self) -> Ventilation: diff --git a/backend/documents_parser/tests/test_pashub_sap_accuracy.py b/backend/documents_parser/tests/test_pashub_sap_accuracy.py index 8db79ed75..8078a1fb0 100644 --- a/backend/documents_parser/tests/test_pashub_sap_accuracy.py +++ b/backend/documents_parser/tests/test_pashub_sap_accuracy.py @@ -19,16 +19,19 @@ a **hybrid** gate: * `test_pashub_fixture_computes` — per fixture: the extractor must produce a calculable `EpcPropertyData`. Any *unexpected* exception is a hard fail (a - real extractor/mapper bug on that property). The one *known* gap — the - pashub mapper not resolving main heating `main_fuel_type` to a SAP fuel - code (`MissingMainFuelType`) — is `xfail`ed dynamically, so these flip to - passing automatically once that mapper fix lands. + real extractor/mapper bug on that property). The one *known* remaining gap — + the pashub mapper not coding a main-heating `main_heating_control` label to a + SAP code — is `xfail`ed dynamically (see `_KNOWN_GAPS` for its two forms), so + these flip to passing once that mapper fix lands. The main-fuel-code gap + (`Bulk LPG`, and the `Fuel:`-form Mains Gas the extractor was dropping) was + closed in #1558, so `MissingMainFuelType` is *no longer* swallowed — a + fixture that regresses to a dropped fuel now hard-fails. * `test_pashub_sap_accuracy_aggregate` — over the fixtures that compute: fraction within 0.5 SAP of `pre_sap`, and MAE. Ratcheting floor/ceiling (never loosen), mirroring the corpus gauge. While every fixture is blocked - on the main-fuel-code gap the aggregate has no data and `xfail`s; once the - mapper fix lands, ratchet the constants below to the observed values. + on the main-heating-control gap the aggregate has no data and `xfail`s; once + that mapper fix lands, ratchet the constants below to the observed values. """ from __future__ import annotations @@ -44,19 +47,18 @@ import pytest from backend.documents_parser.parser import parse_site_notes_pdf from datatypes.epc.domain.mapper import UnmappedPasHubLabel from domain.sap10_calculator.calculator import Sap10Calculator -from domain.sap10_calculator.exceptions import MissingMainFuelType, UnmappedSapCode +from domain.sap10_calculator.exceptions import UnmappedSapCode -# Known in-progress pashub main-heating string→SAP-code mapper gaps. Some -# strict-raise at the mapper boundary during parse (`UnmappedPasHubLabel`, e.g. -# an unmapped "Bulk LPG" fuel), others at calculate time when the calculator -# meets a raw label it can't code (`UnmappedSapCode` for emitter / control, -# `MissingMainFuelType` for an unresolved fuel). Fixed field-by-field (mains-gas -# fuel code landed in #1554); each fix flips its fixtures from xfail to computing. -_KNOWN_GAPS: tuple[type[Exception], ...] = ( - MissingMainFuelType, - UnmappedSapCode, - UnmappedPasHubLabel, -) +# Known in-progress pashub main-heating string→SAP-code mapper gaps, fixed +# field-by-field. The remaining gap is the `main_heating_control` label, which +# surfaces in one of two forms: `UnmappedSapCode` (raw label reaches the +# calculator) or, once the control mapper (#1557) lands, `UnmappedPasHubLabel` +# (strict-raised at the boundary for a control label not yet mapped — e.g. the +# cohort's community-heating charging control). Both xfail until that fix lands. +# The main-fuel-code gaps are closed (mains-gas code #1554; "Bulk LPG" + the +# `Fuel:`-form Mains Gas the extractor was dropping #1558), so `MissingMainFuelType` +# is deliberately *not* here — a fuel-drop regression hard-fails instead of hiding. +_KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel) _FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy" _MANIFEST_PATH = _FIXTURES_DIR / "manifest.json" @@ -73,8 +75,8 @@ _MIN_WITHIN_HALF: float = 0.134 _MAX_SAP_MAE: float = 2.652 _KNOWN_GAP_REASON = ( - "pashub `from_site_notes` mapper does not int-code a main-heating string " - "(fuel / emitter / control) to a SAP code; fix in progress field-by-field" + "pashub `from_site_notes` mapper does not int-code a main-heating " + "`main_heating_control` label to a SAP code; fix in progress field-by-field" ) @@ -95,7 +97,7 @@ class PashubFixture: class Outcome: """Result of running one fixture through the pipeline.""" - blocked: bool # blocked on the known main-fuel-code gap (xfail territory) + blocked: bool # blocked on the known main-heating-control gap (xfail territory) reason: Optional[str] sap_continuous: Optional[float] diff: Optional[float] # abs(our SAP - pre_sap) @@ -117,8 +119,9 @@ _IDS: list[str] = [f.deal_id for f in _FIXTURES] def _evaluate(deal_id: str) -> Outcome: """Run one fixture end-to-end. Shared across the two tests via the cache. - Only `MissingMainFuelType` is swallowed (the known gap). Every other - exception propagates — a real extractor/mapper bug the caller must surface. + Only `UnmappedSapCode` is swallowed (the known control-label gap). Every + other exception propagates — a real extractor/mapper bug the caller must + surface. """ fixture = _BY_ID[deal_id] try: @@ -167,7 +170,7 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No if not diffs: pytest.xfail( f"no fixture computes a SAP score " - f"({blocked} blocked on the main-fuel-code gap, {errored} errored); " + f"({blocked} blocked on the main-heating-control gap, {errored} errored); " f"{_KNOWN_GAP_REASON}" ) diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 947d5e6ca..e71dfc1c9 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -6042,12 +6042,7 @@ def _map_sap_heating( main = heating.main_heating secondary = heating.secondary_heating - # secondary_fuel_type is an int code in the domain model; we can't map a - # site-notes string directly, so leave it None unless there is secondary heating. - # The string fuel type is preserved via sap_heating when needed. - secondary_fuel_type = ( - None if secondary.secondary_fuel == "No Secondary Heating" else None - ) + secondary_fuel_type = _pashub_secondary_fuel_code(secondary.secondary_fuel) shower_outlets = ( ShowerOutlets( @@ -6071,6 +6066,20 @@ def _map_sap_heating( ) ) + # Water heating is only typed for cylinder dwellings; a combi leaves the WHC + # None so HW inherits the main system (issue #1565). + cylinder_present = heating.water_heating.cylinder_size != "No Cylinder" + water_heating_code = ( + _pashub_water_heating_code(heating.water_heating.system) + if cylinder_present + else None + ) + water_heating_fuel = ( + _PASHUB_STANDARD_ELECTRICITY_FUEL + if water_heating_code == _PASHUB_WHC_ELECTRIC_IMMERSION + else None + ) + return SapHeating( instantaneous_wwhrs=InstantaneousWwhrs(), main_heating_details=[ @@ -6094,12 +6103,16 @@ def _map_sap_heating( shower_outlets=shower_outlets, cylinder_size=( heating.water_heating.cylinder_size - if heating.water_heating.cylinder_size != "No Cylinder" + if cylinder_present else None ), cylinder_insulation_type=heating.water_heating.insulation_type, cylinder_insulation_thickness_mm=heating.water_heating.insulation_thickness_mm, - immersion_heating_type=heating.water_heating.immersion_type, + water_heating_code=water_heating_code, + water_heating_fuel=water_heating_fuel, + immersion_heating_type=_pashub_immersion_type_code( + heating.water_heating.immersion_type, cylinder_present + ), ) @@ -7114,9 +7127,44 @@ class UnmappedPasHubLabel(ValueError): # (ADR-0015) rather than being silently mis-billed downstream. _PASHUB_MAIN_FUEL_TO_SAP10: Dict[str, int] = { "Mains gas": 26, # matches the Elmhurst mapper's mains-gas code + # The newer PasHub site-note form lodges the fuel under a `Fuel:` label with + # a capitalised value; same fuel, same code as "Mains gas" above (#1558). + "Mains Gas": 26, + # Bulk LPG → Table 32 code 2 (bulk LPG); 27 = epc_codes `main_fuel` "LPG + # (not community)", the same code the Elmhurst map pins for "Bulk LPG" + # (see `_ELMHURST_MAIN_FUEL_TO_SAP10`). Last residual cohort label that + # parse-raised `UnmappedPasHubLabel` (#1558). + "Bulk LPG": 27, + # Room-heater dual-fuel appliance (mineral + wood). 10 = epc_codes + # `main_fuel` "dual fuel (mineral and wood)" (Table 32 code 10), the code + # the landlord-override and prediction paths use for the same fuel. Surfaced + # once the extractor captured the `Fuel:`-form label it had been dropping. + "Dual fuel appliance (mineral and wood)": 10, "Electricity": 30, # the standard-electricity fuel code } +# PasHub surveyed secondary-heating `secondary_fuel` label → SAP 10.2 Table 32 +# fuel code that `secondary_fuel_type` (an int in the domain model) consumes. +# "No Secondary Heating" is the no-secondary-heater shape and maps to None, +# not a fuel code. +_PASHUB_SECONDARY_FUEL_TO_SAP10: Dict[str, int] = { + "Electricity": 30, # standard electricity, Table 32 code 30 +} + + +def _pashub_secondary_fuel_code(fuel_label: str) -> Optional[int]: + """Resolve a PasHub surveyed secondary-heating Fuel label to a SAP10 + Table 32 fuel code at the mapper boundary (ADR-0015). "No Secondary + Heating" is the no-secondary-heater shape and maps to None; a non-empty + label the lookup does not cover strict-raises `UnmappedPasHubLabel` so + the gap is fixed here, never silently mis-billed downstream.""" + if not fuel_label or fuel_label == "No Secondary Heating": + return None + code = _PASHUB_SECONDARY_FUEL_TO_SAP10.get(fuel_label) + if code is None: + raise UnmappedPasHubLabel("secondary fuel", fuel_label) + return code + def _pashub_main_fuel_code(fuel_label: str) -> Union[int, str]: """Resolve a PasHub surveyed main-heating Fuel label to a SAP10 fuel code @@ -7322,6 +7370,48 @@ def _pashub_glazing_type_int(label: Optional[str]) -> Union[int, str]: return _PASHUB_GLAZING_TYPE_TO_SAP10[key] +# PasHub water-heating `system` label → SAP10 water-heating code (WHC) that the +# calculator's water cascade keys off (cert_to_inputs: `_WHC_FROM_MAIN_HEATING` +# 901 inherits the main fuel; `_WHC_ELECTRIC_IMMERSION` 903; heat-network water +# 950). Only lodged for cylinder dwellings — combi dwellings leave it None and HW +# inherits from the main system (issue #1565). +_PASHUB_WATER_HEATING_SYSTEM_TO_SAP10: Dict[str, int] = { + "From main heating 1": 901, + "Electric immersion": 903, + "Hot water only community scheme - boilers": 950, +} +_PASHUB_WHC_ELECTRIC_IMMERSION: Final[int] = 903 +_PASHUB_STANDARD_ELECTRICITY_FUEL: Final[int] = 30 # Table 32 standard electricity + + +def _pashub_water_heating_code(system_label: Optional[str]) -> Optional[int]: + """Resolve a PasHub water-heating `system` label to its SAP10 WHC code. Blank + is "no lodging" (None); an unmapped label strict-raises `UnmappedPasHubLabel`. + Call only for cylinder dwellings (#1565).""" + if system_label is None or not system_label.strip(): + return None + key = system_label.strip() + if key not in _PASHUB_WATER_HEATING_SYSTEM_TO_SAP10: + raise UnmappedPasHubLabel("water heating system", system_label) + return _PASHUB_WATER_HEATING_SYSTEM_TO_SAP10[key] + + +def _pashub_immersion_type_code( + immersion_label: Optional[str], cylinder_present: bool +) -> Optional[int]: + """Resolve a PasHub `immersion_type` label to the SAP10 `immersion_heating_type` + code, reusing the Elmhurst map (1 = dual, 2 = single, RdSAP 10 §10.5). None + when no cylinder or no immersion lodged; unmapped label strict-raises.""" + if not cylinder_present or immersion_label is None: + return None + key = immersion_label.strip() + if key in ("", "None"): + return None + if key not in _ELMHURST_IMMERSION_TYPE_LABEL_TO_SAP10: + raise UnmappedPasHubLabel("immersion type", immersion_label) + return _ELMHURST_IMMERSION_TYPE_LABEL_TO_SAP10[key] + + def _resolve_elmhurst_underfloor_subtype( main_floor: ElmhurstFloorDetails, main_age_band: str, diff --git a/datatypes/epc/domain/tests/test_from_site_notes.py b/datatypes/epc/domain/tests/test_from_site_notes.py index 657c50033..36d482ce8 100644 --- a/datatypes/epc/domain/tests/test_from_site_notes.py +++ b/datatypes/epc/domain/tests/test_from_site_notes.py @@ -113,6 +113,41 @@ class TestWallConstructionCoding: EpcPropertyDataMapper.from_site_notes(survey) +class TestSecondaryFuelTypeCoding: + """`from_site_notes` must int-code the PAS Hub secondary-heating + `secondary_fuel` label to the SAP 10.2 Table 32 `secondary_fuel_type` + code, not leave it hardcoded `None`. ~67/205 Guinness cohort properties + lodge a "Panel, convector or radiant heaters" secondary heater whose + fuel was silently dropped (issue #1564). + """ + + @pytest.mark.parametrize( + "label, expected_code", + [ + ("Electricity", 30), # standard electricity, Table 32 code 30 + ("No Secondary Heating", None), # no secondary heater lodged + ("", None), # no lodging + ], + ) + def test_label_maps_to_sap_code( + self, label: str, expected_code: Optional[int] + ) -> None: + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["secondary_heating"]["secondary_fuel"] = label + survey = from_dict(PasHubRdSapSiteNotes, data) + result = EpcPropertyDataMapper.from_site_notes(survey) + assert result.sap_heating.secondary_fuel_type == expected_code + + def test_unknown_label_strict_raises(self) -> None: + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["secondary_heating"][ + "secondary_fuel" + ] = "Bulk LPG" + survey = from_dict(PasHubRdSapSiteNotes, data) + with pytest.raises(UnmappedPasHubLabel): + EpcPropertyDataMapper.from_site_notes(survey) + + class TestWallInsulationTypeCoding: """`from_site_notes` must int-code the PAS Hub `walls_insulation_type` label to the SAP10 wall-insulation code `u_wall` consumes (matching the Elmhurst @@ -188,6 +223,97 @@ class TestGlazingTypeCoding: EpcPropertyDataMapper.from_site_notes(survey) +class TestMainFuelCoding: + """`from_site_notes` must int-code the PAS Hub main-heating `fuel` label to the + SAP10 fuel code the calculator's cascade routes via, not copy the raw label. A + raw label reaches the calculator as `MissingMainFuelType`; an unmapped one + strict-raises `UnmappedPasHubLabel` at the boundary. `Bulk LPG` was the last + residual label parse-raising in the Guinness cohort (issue #1558); `Mains Gas` + is the capitalised variant the newer `Fuel:` site-note form lodges. + """ + + @pytest.mark.parametrize( + "label, expected_code", + [ + ("Mains gas", 26), # standard mains-gas fuel code (landed #1554) + ("Mains Gas", 26), # `Fuel:` form capitalisation variant → same code + ("Bulk LPG", 27), # bulk LPG (Table 32 code 2), matches Elmhurst map + # Room-heater dual-fuel appliance; `Fuel:`-form label the extractor + # was dropping. 10 = epc_codes `main_fuel` dual fuel (mineral + wood). + ("Dual fuel appliance (mineral and wood)", 10), + ("Electricity", 30), # standard-electricity fuel code + ], + ) + def test_label_maps_to_sap_code(self, label: str, expected_code: int) -> None: + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["main_heating"]["fuel"] = label + survey = from_dict(PasHubRdSapSiteNotes, data) + result = EpcPropertyDataMapper.from_site_notes(survey) + assert result.sap_heating.main_heating_details[0].main_fuel_type == expected_code + + def test_unknown_label_strict_raises(self) -> None: + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["main_heating"]["fuel"] = "Fusion reactor" + survey = from_dict(PasHubRdSapSiteNotes, data) + with pytest.raises(UnmappedPasHubLabel): + EpcPropertyDataMapper.from_site_notes(survey) +class TestWaterHeatingCoding: + """`from_site_notes` must int-code the PAS Hub water-heating fields for + **cylinder** dwellings — `water_heating_code` (WHC), `water_heating_fuel`, + `immersion_heating_type` — not leave them None/raw. Combi dwellings (no + cylinder) keep the None default (HW inherits from the main system), so they + are untouched (issue #1565). example1 is a cylinder dwelling. + """ + + def _survey(self, **water: object) -> PasHubRdSapSiteNotes: + data = load("pashub_rdsap_site_notes_example1.json") + data["heating_and_hot_water"]["water_heating"].update(water) + return from_dict(PasHubRdSapSiteNotes, data) + + def test_cylinder_from_main_maps_whc_901(self) -> None: + result = EpcPropertyDataMapper.from_site_notes( + self._survey(system="From main heating 1", cylinder_size="Normal (90-130 litres)") + ) + assert result.sap_heating.water_heating_code == 901 + assert result.sap_heating.water_heating_fuel is None + + def test_electric_immersion_maps_903_electric_fuel_and_single_immersion(self) -> None: + result = EpcPropertyDataMapper.from_site_notes( + self._survey( + system="Electric immersion", + cylinder_size="Normal (90-130 litres)", + immersion_type="Single", + ) + ) + assert result.sap_heating.water_heating_code == 903 + assert result.sap_heating.water_heating_fuel == 30 # standard electricity + assert result.sap_heating.immersion_heating_type == 2 # single + + def test_community_scheme_maps_950(self) -> None: + result = EpcPropertyDataMapper.from_site_notes( + self._survey( + system="Hot water only community scheme - boilers", + cylinder_size="Normal (90-130 litres)", + ) + ) + assert result.sap_heating.water_heating_code == 950 + + def test_combi_leaves_whc_none(self) -> None: + result = EpcPropertyDataMapper.from_site_notes( + self._survey(system="From main heating 1", cylinder_size="No Cylinder") + ) + assert result.sap_heating.water_heating_code is None + + def test_unknown_cylinder_system_strict_raises(self) -> None: + with pytest.raises(UnmappedPasHubLabel): + EpcPropertyDataMapper.from_site_notes( + self._survey( + system="Perpetual motion boiler", + cylinder_size="Normal (90-130 litres)", + ) + ) + + class TestFromSiteNotesExample1: """ Fixture: pashub_rdsap_site_notes_example1.json @@ -553,6 +679,7 @@ class TestFromSiteNotesExample1: cylinder_size="Normal (90-130 litres)", cylinder_insulation_type="Factory fitted", cylinder_insulation_thickness_mm=12, + water_heating_code=901, # cylinder + "From main heating 1" → WHC 901 shower_outlets=ShowerOutlets( shower_outlet=ShowerOutlet( shower_outlet_type="Non-Electric Shower"