Merge pull request #1609 from Hestia-Homes/feat/1596-pashub-two-clusters

PasHub SAP accuracy: three extraction silent-drop fixes (within-0.5 23.9% → 50.2%)

backend/documents_parser/tests/test_end_to_end.py

@@ -69,6 +69,10 @@ class TestPdfToEpcPropertyData:
                         shower_outlet_type="Non-Electric Shower"
                     ),
                 ),
+                # water_use: one 'Non-Electric Shower' (mixer), one bath (#1598)
+                number_baths=1,
+                electric_shower_count=0,
+                mixer_shower_count=1,
             ),
             sap_windows=[
                 SapWindow(
@@ -213,7 +217,7 @@ class TestPdfToEpcPropertyData:
                     wall_thickness_mm=310,
                     roof_insulation_location="Joists",
                     roof_insulation_thickness=100,
-                    floor_type="Ground Floor",
+                    floor_type="Ground floor",  # normalized (#1599)
                     floor_construction_type="Solid",
                     floor_insulation_type_str="As Built",
                     floor_u_value_known=False,

backend/documents_parser/tests/test_pashub_sap_accuracy.py

@@ -88,7 +88,16 @@ _MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"
 # observed 20.0% (41/205).
 # Ratcheted 0.19 -> 0.23 by the window-orientation fix (see the MAE note
 # below): observed 23.9% (49/205) — the campaign's largest within-0.5 move.
-_MIN_WITHIN_HALF: float = 0.23
+# Ratcheted 0.23 -> 0.50 by the #1596 extraction bundle (#1598 electric-shower
+# + bath counts never threaded into `sap_heating`; #1599 floor label "Ground
+# Floor" normalized to the calculator's "Ground floor" so the §5(12) suspended-
+# timber infiltration term fires; #1600 water-heating "None" coded to the RdSAP
+# §10.7 WHC 999 default instead of inheriting the gas combi). Observed 50.2%
+# (103/205) — the campaign's largest move. All three are silent-drop fixes the
+# gov-API/Elmhurst paths already apply; verified against Khalim's ground truth
+# (16 Stillwater 78.2 -> 75.7 vs 76; 58 Hackle 67.7 -> ~58 vs 59). #1601 (upper-
+# floor +0.25 m) is HELD — it empirically over-corrects this cohort.
+_MIN_WITHIN_HALF: float = 0.50
 # Ratcheted 2.71 -> 2.64 by #1590 bug 2 (roof "Insulation At: None" -> explicit
 # zero thickness): observed MAE 2.632 — the two ~7-SAP roof over-raters
 # (507665533138 / 507670893756) correcting is exactly 14/205 ≈ 0.07 of MAE.
@@ -143,35 +152,17 @@ _MIN_WITHIN_HALF: float = 0.23
 # 1.4103; re-tighten when the water-only branch is fixed correctly under #1592
 # (the cylinder-insulation MAPPER fix is retained, so the revert is partial).
 # DELIBERATELY LOOSENED 1.411 -> 1.415 by the #1592 sub-task B water-only DHW
-# fix — the SECOND sanctioned exception to "ratchets never loosen", and the
-# note above expected to RE-TIGHTEN here, so this needs @KhalimCK's explicit
-# call before merge.
-#
-# The fix is spec-verified against the SAP 10.2 PDF (not ticket prose): §4.3
-# p.24 "The efficiency for water heating is incorporated in the price of heat
-# from a heat network in Table 12"; C4 p.53 scopes the plant efficiency to
-# "the calculation of CO2 emissions and Primary Energy"; C6 p.54 + worksheet
-# (310) = (64) × (305a) × (306) carry no efficiency term. The dwelling-side
-# `plant_eff / DLF` divisor was applying it a SECOND time.
-#
-# Exactly ONE cohort property is WHC 950 — 507644414148 (16 Bingley Close),
-# +5.946 -> +6.818 = 0.872/205 = +0.0043 MAE, the whole movement. within-0.5
-# UNCHANGED at 23.9% (49/205); mean signed +1.050 -> +1.054. Observed 1.41458.
-#
-# Why this is not a regression signal: Bingley's `pre_sap` E52 is Khalim-
-# verified correct (2026-07-14), but its ~+6 residual is a KNOWN extraction bug
-# still being hunted (see HANDOVER_PASHUB_EXTRACTOR_ACCURACY.md: "community-
-# heating residual — ... community standing charge/tariff detail, or DLF
-# 'Unknown' distribution type"). This hybrid gauge conflates that extraction
-# error with the SAP-10.2-vs-lodged-2012-RdSAP engine offset, so per #1592
-# Problem 2 it CANNOT adjudicate a calculator fix — the same reasoning the
-# orientation exception above rests on. The gov-API RdSAP corpus, which gauges
-# the calculator alone, IMPROVES on this change (within-0.5 78.7% -> 78.8%,
-# MAE 0.6273 -> 0.6272; exactly 3 certs move, all WHC 950).
-#
-# Re-tighten under #1592 sub-task C, once an accredited Elmhurst 10.2 worksheet
-# pins WHC-950 — Bingley is the natural capture candidate.
-_MAX_SAP_MAE: float = 1.415
+# fix (16 Bingley Close, WHC 950, +5.946 -> +6.818 = +0.0043 MAE) — a sanctioned
+# "never loosen" exception that expected re-tightening here.
+# RE-TIGHTENED 1.415 -> 0.653 by the #1596 extraction bundle (#1598 electric-
+# shower + bath counts; #1599 floor label "Ground Floor" -> "Ground floor" so the
+# §5(12) suspended-timber infiltration fires; #1600 water-heating "None" -> RdSAP
+# §10.7 WHC 999 — see the within-0.5 note above), which SUPERSEDES the #1592
+# loosening: observed MAE 0.6517 on the merged tree (with the water-only DHW fix,
+# 16 Bingley still +6.818). The three silent-drop fixes the gov-API/Elmhurst
+# paths already apply; the electric-shower drop (100/205) was the dominant lever.
+# #1601 (upper-floor +0.25 m) is HELD — it empirically over-corrects this cohort.
+_MAX_SAP_MAE: float = 0.653
 
 _KNOWN_GAP_REASON = (
     "pashub `from_site_notes` mapper does not int-code a main-heating "

datatypes/epc/domain/mapper.py

@@ -383,7 +383,9 @@ class EpcPropertyDataMapper:
             ),
             sap_building_parts=sap_building_parts,
             solar_water_heating=renewables.solar_hot_water,
-            has_hot_water_cylinder=heating.water_heating.cylinder_size != "No Cylinder",
+            has_hot_water_cylinder=_pashub_cylinder_present(
+                heating.water_heating.cylinder_size
+            ),
             has_fixed_air_conditioning=ventilation.has_fixed_air_conditioning,
             wet_rooms_count=0,  # no equivalent in site notes
             extensions_count=general.number_of_extensions,
@@ -5948,6 +5950,12 @@ def _extract_age_band(age_range: str) -> str:
 
 
 def _map_floor_dimensions(floors: List[FloorMeasurement]) -> List[SapFloorDimension]:
+    # NOTE (#1601, HELD): the gov-API/Elmhurst mappers add a 0.25 m joist-void to
+    # UPPER-storey heights. Adding it here is spec-plausible but EMPIRICALLY
+    # over-corrects the PAS Hub cohort (within-0.5 50%→18%, moves 16 Stillwater
+    # away from its verified 76) once the shower/floor fixes land — pashub's
+    # surveyed height likely already behaves as a storey height. Held pending
+    # ground truth on PAS Hub's height semantics; left as the raw surveyed value.
     return [
         SapFloorDimension(
             room_height_m=floor.height_m,
@@ -6083,6 +6091,22 @@ def _pashub_room_in_roof(
     )
 
 
+# PAS Hub lodges the ground-floor type as "Ground Floor" (capital F). The
+# calculator's suspended-timber floor-infiltration gate (RdSAP 10 §5(12),
+# `_has_suspended_timber_floor_per_spec`) compares against the canonical
+# "Ground floor" (lowercase f) that the gov-API path emits, so the capital-F
+# label silently dropped the 0.2-ACH infiltration term for every PAS Hub
+# suspended-timber floor. Normalize to the canonical spelling at the mapper
+# boundary; any other floor-type label passes through unchanged (#1599).
+_CANONICAL_GROUND_FLOOR_TYPE: Final[str] = "Ground floor"
+
+
+def _pashub_floor_type(label: str) -> str:
+    if label.strip().lower() == _CANONICAL_GROUND_FLOOR_TYPE.lower():
+        return _CANONICAL_GROUND_FLOOR_TYPE
+    return label
+
+
 def _map_main_building_part(
     construction: BuildingConstruction,
     measurements: BuildingMeasurements,
@@ -6118,7 +6142,7 @@ def _map_main_building_part(
         roof_construction_type=roof.construction_type or None,
         roof_insulation_location=roof_location,
         roof_insulation_thickness=roof_thickness,
-        floor_type=floor.floor_type,
+        floor_type=_pashub_floor_type(floor.floor_type),
         floor_construction_type=floor.floor_construction,
         floor_insulation_type_str=floor.floor_insulation_type,
         floor_u_value_known=floor.floor_u_value_known,
@@ -6179,6 +6203,12 @@ def _map_sap_window(window: Window) -> SapWindow:
     )
 
 
+# PAS Hub lodges a shower outlet's type as this exact label (capital S) for an
+# electric shower; anything else ("Non-Electric Shower") is a mixer draw. The
+# Elmhurst sibling uses the lowercase "Electric shower" spelling.
+_PASHUB_ELECTRIC_SHOWER_LABEL: Final[str] = "Electric Shower"
+
+
 def _map_sap_heating(
     heating: HeatingAndHotWater, ventilation: Ventilation, water_use: WaterUse
 ) -> SapHeating:
@@ -6197,6 +6227,21 @@ def _map_sap_heating(
         else None
     )
 
+    # SAP 10.2 Appendix J routes electric showers via §J line 64a (their own
+    # kWh stream) and treats mixer showers as drawing from the HW system, so
+    # the cascade needs the actual per-type counts. Leaving them None made it
+    # default to 1 gas mixer / 0 electric, billing electric-shower hot water
+    # off the cheap main system and over-rating SAP (issue #1598). Tally the
+    # surveyed outlet list by the PAS Hub label ("Electric Shower" / "Non-
+    # Electric Shower"), mirroring the gov-API `_count_shower_outlets_by_type`
+    # and Elmhurst paths; a zero-shower lodging resolves to (0, 0).
+    electric_shower_count = sum(
+        1 for s in water_use.showers if s.outlet_type == _PASHUB_ELECTRIC_SHOWER_LABEL
+    )
+    mixer_shower_count = sum(
+        1 for s in water_use.showers if s.outlet_type != _PASHUB_ELECTRIC_SHOWER_LABEL
+    )
+
     _ELECTRIC_SYSTEM_TYPES = {"electric storage heaters", "electric underfloor heating"}
     _raw_fuel = main.fuel.split(", ")[0] if main.fuel else ""
     fuel_type = (
@@ -6210,13 +6255,17 @@ def _map_sap_heating(
     )
 
     # Water heating is only typed for cylinder dwellings; a combi leaves the WHC
-    # None so HW inherits the main system (issue #1565).
-    cylinder_present = heating.water_heating.cylinder_size != "No Cylinder"
-    water_heating_code = (
-        _pashub_water_heating_code(heating.water_heating.system)
-        if cylinder_present
-        else None
-    )
+    # None so HW inherits the main system (issue #1565). A `Water Heating Type:
+    # None` lodgement is the RdSAP 10 §10.7 "no water heating system" signal and
+    # must NOT inherit the (cheap) main system — code it WHC 999 so the
+    # calculator applies its electric-immersion + Table 28/29 defaults (#1600).
+    cylinder_present = _pashub_cylinder_present(heating.water_heating.cylinder_size)
+    if heating.water_heating.type.strip() == _PASHUB_NO_WATER_HEATING_TYPE:
+        water_heating_code: Optional[int] = _PASHUB_WHC_NO_WATER_HEATING_SYSTEM
+    elif cylinder_present:
+        water_heating_code = _pashub_water_heating_code(heating.water_heating.system)
+    else:
+        water_heating_code = None
     water_heating_fuel = (
         _PASHUB_STANDARD_ELECTRICITY_FUEL
         if water_heating_code == _PASHUB_WHC_ELECTRIC_IMMERSION
@@ -6265,6 +6314,9 @@ def _map_sap_heating(
             heating.secondary_heating.secondary_system
         ),
         shower_outlets=shower_outlets,
+        number_baths=water_use.number_of_baths,
+        electric_shower_count=electric_shower_count,
+        mixer_shower_count=mixer_shower_count,
         cylinder_size=(
             _pashub_cylinder_size_code(
                 heating.water_heating.cylinder_size,
@@ -7826,6 +7878,24 @@ _PASHUB_WATER_HEATING_SYSTEM_TO_SAP10: Dict[str, int] = {
 }
 _PASHUB_WHC_ELECTRIC_IMMERSION: Final[int] = 903
 _PASHUB_STANDARD_ELECTRICITY_FUEL: Final[int] = 30  # Table 32 standard electricity
+# RdSAP 10 §10.7 (PDF p.55) "No water heating system" — WHC 999. When PAS Hub
+# lodges `Water Heating Type: None` (no system identified), the calculator's
+# `_apply_rdsap_no_water_heating_system_default` substitutes electric immersion
+# + the Table 28/29 cylinder defaults, matching the gov-API/Elmhurst path
+# (mirrors `cert_to_inputs._WHC_NO_WATER_HEATING_SYSTEM`; leaf-module convention
+# — the mapper keeps a local copy rather than importing the private constant).
+_PASHUB_WHC_NO_WATER_HEATING_SYSTEM: Final[int] = 999
+# PAS Hub label for the "no water heating system" lodgement (Water Heating Type).
+_PASHUB_NO_WATER_HEATING_TYPE: Final[str] = "None"
+
+
+def _pashub_cylinder_present(cylinder_size: Optional[str]) -> bool:
+    """A PAS Hub dwelling has a hot-water cylinder iff a non-blank cylinder size
+    is lodged that is not "No Cylinder". A blank/None size is NOT a cylinder — a
+    combi lodging no "Cylinder Size:" line was previously billed a phantom
+    cylinder (storage + primary loss). "No Access" stays present — the cylinder
+    exists, it just wasn't measured (issue #1594)."""
+    return bool(cylinder_size) and cylinder_size != "No Cylinder"
 
 
 def _pashub_water_heating_code(system_label: Optional[str]) -> Optional[int]:

datatypes/epc/domain/tests/test_from_site_notes.py

@@ -1109,6 +1109,106 @@ class TestWaterHeatingCoding:
             )
 
 
+class TestNoWaterHeatingSystemAndCylinderPresence:
+    """A PAS Hub `Water Heating Type: None` lodgement is the RdSAP 10 §10.7
+    "no water heating system" signal → WHC 999 (the calculator then applies its
+    electric-immersion + Table 28/29 defaults), NOT the gas-combi inheritance
+    (issue #1600 / 58 Hackle St). Separately, `has_hot_water_cylinder` must be
+    False for a blank/None cylinder size (a combi lodging no "Cylinder Size:"
+    line was billed a phantom cylinder), while "No Access" stays present
+    (issue #1594).
+    """
+
+    def _survey(
+        self, *, wh_type: str = "Regular", system: str = "From main heating 1",
+        cylinder_size: str = "No Cylinder",
+    ) -> PasHubRdSapSiteNotes:
+        data = load("pashub_rdsap_site_notes_example1.json")
+        wh = data["heating_and_hot_water"]["water_heating"]
+        wh["type"] = wh_type
+        wh["system"] = system
+        wh["cylinder_size"] = cylinder_size
+        return from_dict(PasHubRdSapSiteNotes, data)
+
+    def test_water_heating_type_none_maps_whc_999(self) -> None:
+        # 58 Hackle: Water Heating Type "None", blank system + cylinder.
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(wh_type="None", system="", cylinder_size="")
+        )
+        assert result.sap_heating.water_heating_code == 999
+
+    def test_blank_cylinder_size_has_no_cylinder(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(wh_type="None", system="", cylinder_size="")
+        )
+        assert result.has_hot_water_cylinder is False
+
+    def test_explicit_no_cylinder_has_no_cylinder(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(cylinder_size="No Cylinder")
+        )
+        assert result.has_hot_water_cylinder is False
+
+    def test_real_cylinder_size_is_present(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(cylinder_size="Normal (90-130 litres)")
+        )
+        assert result.has_hot_water_cylinder is True
+
+    def test_no_access_cylinder_is_present(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(cylinder_size="No Access")
+        )
+        assert result.has_hot_water_cylinder is True
+
+
+class TestShowerAndBathCounts:
+    """`from_site_notes` must thread the surveyed shower outlets and bath count
+    into `sap_heating.electric_shower_count` / `mixer_shower_count` /
+    `number_baths`. The calculator reads those counts (SAP 10.2 Appendix J
+    routes electric showers to their own kWh stream); leaving them None made
+    the cascade default to 1 gas mixer / 0 electric, billing electric-shower
+    hot water off the cheap main system and over-rating SAP (issue #1598).
+    Mirrors the gov-API (`_count_shower_outlets_by_type`) and Elmhurst paths.
+    example1 lodges one 'Non-Electric Shower' and one bath.
+    """
+
+    def _survey(
+        self, showers: list[str], baths: int = 1
+    ) -> PasHubRdSapSiteNotes:
+        data = load("pashub_rdsap_site_notes_example1.json")
+        data["water_use"]["showers"] = [
+            {"id": i + 1, "outlet_type": t} for i, t in enumerate(showers)
+        ]
+        data["water_use"]["number_of_baths"] = baths
+        return from_dict(PasHubRdSapSiteNotes, data)
+
+    def test_electric_shower_counted(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(["Electric Shower"])
+        )
+        assert result.sap_heating.electric_shower_count == 1
+        assert result.sap_heating.mixer_shower_count == 0
+
+    def test_non_electric_shower_is_mixer(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(["Non-Electric Shower"])
+        )
+        assert result.sap_heating.electric_shower_count == 0
+        assert result.sap_heating.mixer_shower_count == 1
+
+    def test_no_showers_counts_zero(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(self._survey([]))
+        assert result.sap_heating.electric_shower_count == 0
+        assert result.sap_heating.mixer_shower_count == 0
+
+    def test_number_baths_threaded(self) -> None:
+        result = EpcPropertyDataMapper.from_site_notes(
+            self._survey(["Electric Shower"], baths=2)
+        )
+        assert result.sap_heating.number_baths == 2
+
+
 class TestMainHeatingEfficiencySource:
     """`from_site_notes` must carry the surveyed PCDB `product_id` onto
     `main_heating_index_number`, so the calculator reads the actual appliance's
@@ -1360,7 +1460,7 @@ class TestFromSiteNotesExample1:
         )
 
     def test_floor_height(self, result: EpcPropertyData) -> None:
-        # floors[0].height_m: 2.37
+        # floors[0].height_m: 2.37 (Floor 1, upper)
         assert (
             result.sap_building_parts[0].sap_floor_dimensions[0].room_height_m == 2.37
         )
@@ -1513,6 +1613,10 @@ class TestFromSiteNotesExample1:
                         shower_outlet_type="Non-Electric Shower"
                     ),
                 ),
+                # water_use: one 'Non-Electric Shower' (mixer), one bath (#1598)
+                number_baths=1,
+                electric_shower_count=0,
+                mixer_shower_count=1,
             ),
             # Windows
             sap_windows=[
@@ -1610,7 +1714,7 @@ class TestFromSiteNotesExample1:
                     roof_construction_type="Pitched roof (Slates or tiles), Access to loft",
                     roof_insulation_location="Joists",
                     roof_insulation_thickness=100,
-                    floor_type="Ground Floor",
+                    floor_type="Ground floor",  # normalized from "Ground Floor" (#1599)
                     floor_construction_type="Suspended, not timber",
                     floor_insulation_type_str="As Built",
                     floor_u_value_known=False,
@@ -1735,8 +1839,11 @@ class TestFromSiteNotesFloorConstruction:
         return EpcPropertyDataMapper.from_site_notes(survey)
 
     def test_floor_type(self, result: EpcPropertyData) -> None:
-        # building_construction.floor.floor_type: "Ground Floor"
-        assert result.sap_building_parts[0].floor_type == "Ground Floor"
+        # building_construction.floor.floor_type: "Ground Floor" (capital F) is
+        # normalized to the calculator's canonical "Ground floor" so the RdSAP
+        # §5(12) suspended-timber floor-infiltration gate (case-sensitive on
+        # "Ground floor") fires instead of dropping to 0 ACH (issue #1599).
+        assert result.sap_building_parts[0].floor_type == "Ground floor"
 
     def test_floor_construction_type(self, result: EpcPropertyData) -> None:
         # building_construction.floor.floor_construction: "Suspended, not timber"