junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-27 23:35:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into fix/1589-refetch-epc-is-ignored

Browse source
This commit is contained in:
Daniel Roth 2026-07-15 09:45:20 +00:00
commit 093e5ce041
12 changed files with 2242 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
backend/documents_parser/extractor.py
View file

@ -18,6 +18,9 @@ from datatypes.epc.surveys.pashub_rdsap_site_notes import (
MainBuildingMeasurements,
MainHeating,
PasHubRdSapSiteNotes,
PvArrayDetail,
RoomInRoofDetail,
RoomInRoofSurfaceDetail,
Renewables,
RoomCountElements,
RoofSpace,
@ -245,6 +248,7 @@ class PasHubRdSapSiteNotesExtractor:
data, "Party wall construction type:"
)
or "",
dry_lined=self._optional_bool_in(data, "Wall Dry-Lined?"),
)
def _parse_extension_construction(
@ -275,6 +279,7 @@ class PasHubRdSapSiteNotesExtractor:
data, "Party wall construction type:"
)
or "",
dry_lined=self._optional_bool_in(data, "Wall Dry-Lined?"),
)
def extract_building_measurements(self) -> BuildingMeasurements:
@ -435,8 +440,41 @@ class PasHubRdSapSiteNotesExtractor:
hydro=self._bool_in(r_section, "Is the dwelling connected to Hydro?"),
pv_connection=pv_connection,
percent_roof_covered_pv=percent_roof,
pv_diverter_present=self._optional_bool_in(
r_section, "Is there a PV diverter?"
),
pv_arrays=self._pv_arrays_in(r_section),
)
def _pv_arrays_in(self, r_section: List[str]) -> Optional[List[PvArrayDetail]]:
"""The "Photovoltaic array kWp Known? Yes" form lodges a per-system
block instead of the percent-roof estimate: "Number of PV systems?"
then "PV {n}: kWp value / Photovoltaic Pitch / Orientation /
Overshading" per system. Dropped detail silently zeroes the Appendix M
PV credit downstream (issue #1590 bug 1)."""
systems_raw = self._get_in(r_section, "Number of PV systems?")
if systems_raw is None:
return None
arrays: List[PvArrayDetail] = []
for n in range(1, int(systems_raw) + 1):
kwp_raw = self._get_in(r_section, f"PV {n}: kWp value:")
pitch_raw = self._get_in(r_section, f"PV {n}: Photovoltaic Pitch?")
arrays.append(
PvArrayDetail(
kwp=float(kwp_raw.split()[0]) if kwp_raw else None,
pitch_degrees=(
int(pitch_raw.split()[0]) if pitch_raw else None
),
orientation=self._get_in(
r_section, f"PV {n}: Photovoltaic Orientation?"
),
overshading=self._get_in(
r_section, f"PV {n}: Photovoltaic Overshading:"
),
)
)
return arrays or None
def extract_room_count_elements(self) -> RoomCountElements:
rce_section = self._section("Room Count Elements", "Customer Response")
heated_rooms_raw = self._get_in(rce_section, "Number of heated rooms?")
@ -559,6 +597,8 @@ class PasHubRdSapSiteNotesExtractor:
)
or "",
system_type=self._get_in(data, "System type:") or "",
community_heat_source=self._get_in(data, "Heating System (Other):")
or "",
product_id=int(self._get_in(data, "Product Id") or 0),
manufacturer=self._get_in(data, "Manufacturer") or "",
model=self._get_in(data, "Model") or "",
@ -638,6 +678,59 @@ class PasHubRdSapSiteNotesExtractor:
except (ValueError, IndexError):
return None, val
def _parse_rir_surface(
self, data: List[str], prefix: str, with_type: bool = False
) -> Optional[RoomInRoofSurfaceDetail]:
"""One room-in-roof surface's "{prefix} length/height[/type]" lines
("6.64 m" → 6.64). None when the surface isn't lodged."""
length = self._get_in(data, f"{prefix} length:")
height = self._get_in(data, f"{prefix} height:")
if length is None and height is None:
return None
# "{prefix}: Are you able to provide details of the insulation …?" —
# the question wraps onto a second PDF line, so the Yes/No sits at
# offset 2 from the stable first-line key. Absent on legacy forms.
known_raw = self._get_in(
data, f"{prefix}: Are you able to provide details of the", offset=2
)
return RoomInRoofSurfaceDetail(
length_m=float(length.split()[0]) if length else None,
height_m=float(height.split()[0]) if height else None,
gable_type=self._get_in(data, f"{prefix} type:") if with_type else None,
insulation_known=(
None if known_raw is None else known_raw.lower() == "yes"
),
)
def _parse_room_in_roof(self, data: List[str]) -> Optional[RoomInRoofDetail]:
"""The RdSAP §3.10 detailed room-in-roof block lodged under "Are there
rooms in the roof? Yes" — dropped detail bills ~the whole RIR shell as
well-insulated loft (issue #1590 bug 5)."""
if not self._bool_in(data, "Are there rooms in the roof?"):
return None
area_raw = self._get_in(data, "Floor area of room in roof:")
def surfaces(
prefix: str, with_type: bool = False
) -> Optional[List[RoomInRoofSurfaceDetail]]:
out = [
s
for n in (1, 2)
if (s := self._parse_rir_surface(data, f"{prefix} {n}", with_type))
is not None
]
return out or None
return RoomInRoofDetail(
age_range=self._get_in(data, "Roof room age range:"),
floor_area_m2=float(area_raw.split()[0]) if area_raw else None,
gables=surfaces("Gable wall", with_type=True),
slopes=surfaces("Slope"),
common_walls=surfaces("Common wall"),
flat_ceilings=surfaces("Flat ceiling"),
roof_room_type=self._get_in(data, "Roof room type:"),
)
def _parse_roof_space_detail(self, data: List[str]) -> RoofSpaceDetail:
thickness_mm, thickness_str = self._parse_insulation_thickness(
self._get_in(data, "Roofs - Insulation Thickness:")
@ -653,6 +746,7 @@ class PasHubRdSapSiteNotesExtractor:
rooms_in_roof=self._bool_in(data, "Are there rooms in the roof?"),
insulation_thickness_mm=thickness_mm,
insulation_thickness=thickness_str,
room_in_roof=self._parse_room_in_roof(data),
)
def _parse_extension_roof_space(
@ -673,6 +767,7 @@ class PasHubRdSapSiteNotesExtractor:
rooms_in_roof=self._bool_in(data, "Are there rooms in the roof?"),
insulation_thickness_mm=thickness_mm,
insulation_thickness=thickness_str,
room_in_roof=self._parse_room_in_roof(data),
)
def _parse_floor_measurements(self, data: List[str]) -> List[FloorMeasurement]:

18
backend/documents_parser/tests/fixtures/pashub_accuracy/manifest.json vendored
View file

@ -13,8 +13,8 @@
{
"deal_id": "499516101839",
"uprn": null,
"pre_sap_raw": "C73",
"pre_sap_score": 73,
"pre_sap_raw": "C75",
"pre_sap_score": 75,
"pdf": "499516101839.pdf"
},
{
@ -223,8 +223,8 @@
{
"deal_id": "499539910905",
"uprn": null,
"pre_sap_raw": "D58",
"pre_sap_score": 58,
"pre_sap_raw": "B82",
"pre_sap_score": 82,
"pdf": "499539910905.pdf"
},
{
@ -587,8 +587,8 @@
{
"deal_id": "499584755922",
"uprn": null,
"pre_sap_raw": "D67",
"pre_sap_score": 67,
"pre_sap_raw": "F33",
"pre_sap_score": 33,
"pdf": "499584755922.pdf"
},
{
@ -811,8 +811,8 @@
{
"deal_id": "499617935574",
"uprn": null,
"pre_sap_raw": "E53",
"pre_sap_score": 53,
"pre_sap_raw": "E43",
"pre_sap_score": 43,
"pdf": "499617935574.pdf"
},
{
@ -1439,4 +1439,4 @@
"pdf": "507693673709.pdf"
}
]
}
}

8
backend/documents_parser/tests/test_end_to_end.py
View file

@ -388,10 +388,10 @@ class TestPdfToEpcPropertyDataFixture5:
assert result.cfl_fixed_lighting_bulbs_count == 2
def test_secondary_heating_type(self, result: EpcPropertyData) -> None:
assert (
result.sap_heating.secondary_heating_type
== "Panel, convector or radiant heaters"
)
# "Panel, convector or radiant heaters" → SAP10 Table 4a room-heater
# code 691 (int-coded at the mapper boundary; the calculator reads
# the code, not the label).
assert result.sap_heating.secondary_heating_type == 691
def test_electric_shower_outlet_type(self, result: EpcPropertyData) -> None:
assert result.sap_heating.shower_outlets is not None

436
backend/documents_parser/tests/test_extractor.py
View file

@ -21,6 +21,8 @@ from datatypes.epc.surveys.pashub_rdsap_site_notes import (
MainBuildingConstruction,
MainBuildingMeasurements,
MainHeating,
PvArrayDetail,
RoomInRoofSurfaceDetail,
Renewables,
RoomCountElements,
RoofSpace,
@ -196,6 +198,29 @@ class TestBuildingConstruction:
assert construction.extensions is not None
assert construction.extensions[0].id == 1
def test_wall_dry_lined_answers_are_captured(self) -> None:
# Arrange / Act — fixture 7 lodges "Wall Dry-Lined?" per building
# part: Main "No", Extension 1 "Yes", Extension 2 "Yes". Dropped, a
# dry-lined wall is billed at the full uninsulated Table 6 U instead
# of the RdSAP10 §5.8 / Table 14 dry-lining-adjusted value (19% of
# the Guinness accuracy cohort lodges a "Yes").
construction = PasHubRdSapSiteNotesExtractor(
load_text_fixture_7()
).extract_building_construction()
# Assert
assert construction.main_building.dry_lined is False
assert construction.extensions is not None
assert [e.dry_lined for e in construction.extensions] == [True, True]
def test_wall_dry_lined_not_asked_stays_unknown(
self, construction: BuildingConstruction
) -> None:
# Fixture 1 never asks "Wall Dry-Lined?" — unknown, NOT a "No".
assert construction.main_building.dry_lined is None
assert construction.extensions is not None
assert construction.extensions[0].dry_lined is None
def test_full_building_construction(
self, construction: BuildingConstruction
) -> None:
@ -646,6 +671,417 @@ class TestRenewablesPvConnection:
assert renewables.percent_roof_covered_pv == 45
class TestRenewablesPvArrays:
"""When "Photovoltaic array kWp Known?" is Yes, the survey lodges a
per-system block ("Number of PV systems?", then "PV {n}: kWp value /
Photovoltaic Pitch / Orientation / Overshading") instead of the
percent-roof estimate. The extractor must capture it — dropped detail
silently zeroes the Appendix M PV credit (issue #1590 bug 1; verbatim
lines from accuracy fixture deal 507639151843).
"""
# The real kWp-known block, verbatim from the 507639151843 site notes.
_KWP_KNOWN_BLOCK = [
"Photovoltaic array kWp Known?",
"Yes",
"Number of PV systems?",
"1",
"PV 1: kWp value:",
"2.43 kWp",
"PV 1: Photovoltaic Pitch?",
"30 Degrees",
"PV 1: Photovoltaic Orientation?",
"South East",
"PV 1: Photovoltaic Overshading:",
"None or very little",
]
@pytest.fixture
def renewables(self) -> Renewables:
# Arrange — fixture 3's Renewables section with its "kWp Known? No +
# percent roof" block swapped for the real kWp-known block.
lines = load_text_fixture_3()
start = lines.index("Photovoltaic array kWp Known?")
end = lines.index("Number of PV batteries:")
spliced = lines[:start] + self._KWP_KNOWN_BLOCK + lines[end:]
return PasHubRdSapSiteNotesExtractor(spliced).extract_renewables()
def test_pv_array_detail_is_captured(self, renewables: Renewables) -> None:
assert renewables.pv_arrays == [
PvArrayDetail(
kwp=2.43,
pitch_degrees=30,
orientation="South East",
overshading="None or very little",
)
]
def test_pv_diverter_is_captured(self, renewables: Renewables) -> None:
assert renewables.pv_diverter_present is False # fixture 3 lodges "No"
def test_no_pv_block_leaves_arrays_absent(self) -> None:
# Arrange / Act — fixture 3 as lodged (kWp Known? No, percent path).
renewables = PasHubRdSapSiteNotesExtractor(
load_text_fixture_3()
).extract_renewables()
# Assert
assert renewables.pv_arrays is None
class TestMainHeatingCommunityHeatSource:
"""A community-heating dwelling lodges its heat source under "Heating
System (Other):" (e.g. "Community heating - boilers only", verbatim from
accuracy fixture deal 507644414148). Dropped, the mapper cannot lodge the
Table 4a heat-network SAP code and the dwelling is priced as an ordinary
boiler (issue #1590 follow-up).
"""
def test_heat_source_label_is_captured(self) -> None:
# Arrange — fixture 1's heating section with the community block
# spliced in after its "Fuel:" lodgement.
lines = load_text_fixture()
idx = lines.index("System type:")
spliced = (
lines[: idx + 2]
+ ["Heating System (Other):", "Community heating - boilers only"]
+ lines[idx + 2 :]
)
# Act
mh = PasHubRdSapSiteNotesExtractor(
spliced
).extract_heating_and_hot_water().main_heating
# Assert
assert mh.community_heat_source == "Community heating - boilers only"
def test_absent_label_stays_blank(self) -> None:
# Arrange / Act — fixture 1 as lodged (no community block).
mh = PasHubRdSapSiteNotesExtractor(
load_text_fixture()
).extract_heating_and_hot_water().main_heating
# Assert
assert mh.community_heat_source == ""
class TestRoofSpaceRoomInRoof:
""""Are there rooms in the roof? Yes" lodges an RdSAP §3.10 detailed RIR
block (age, floor area, gables/slopes/common walls/flat ceiling as
length × height). The extractor must capture it — dropped, ~32 m² of RIR
surfaces are billed as well-insulated loft (issue #1590 bug 5; lines
verbatim from accuracy fixture deal 499617935574, incl. the mid-block
"Page 7" break).
"""
_RIR_BLOCK = [
"Are there rooms in the roof?",
"Yes",
"Roof room age range:",
"B: 1900 - 1929",
"Floor area of room in roof:",
"32.35 m²",
"Are details of the room in roof known?",
"Yes",
"Are details of the gable walls known?",
"Gable wall 1 & 2",
"Gable wall 1 length:",
"6.64 m",
"Gable wall 1 height:",
"2.74 m",
"Page 7",
"Gable wall 1 U-Value:",
"Unknown",
"Gable wall 1 type:",
"Exposed",
"Gable wall 2 length:",
"6.64 m",
"Gable wall 2 height:",
"2.74 m",
"Gable wall 2 type:",
"Party",
"Are details of the stud walls known?",
"None",
"Are details of the slope walls known?",
"Slope 1 & 2",
"Slope 1 length:",
"4.94 m",
"Slope 1 height:",
"1.73 m",
"Slope 2 length:",
"5 m",
"Slope 2 height:",
"2.95 m",
"Are details of the common walls known?",
"Common wall 1 & 2",
"Common wall 1 length:",
"4.94 m",
"Common wall 1 height:",
"1.45 m",
"Common wall 2 length:",
"4.94 m",
"Common wall 2 height:",
"1.3 m",
"Are details of the flat ceiling known?",
"Flat ceiling 1",
"Flat ceiling 1 length:",
"4.94 m",
"Flat ceiling 1 height:",
"5.17 m",
]
def test_room_in_roof_block_is_captured(self) -> None:
# Arrange — fixture 1's roof space with the RIR block spliced in place
# of its "rooms in the roof? No" lodging.
lines = load_text_fixture()
idx = lines.index("Are there rooms in the roof?")
spliced = lines[:idx] + self._RIR_BLOCK + lines[idx + 2 :]
# Act
detail = PasHubRdSapSiteNotesExtractor(
spliced
).extract_roof_space().main_building
# Assert
assert detail.rooms_in_roof is True
rir = detail.room_in_roof
assert rir is not None
assert rir.age_range == "B: 1900 - 1929"
assert rir.floor_area_m2 == 32.35
assert rir.gables == [
RoomInRoofSurfaceDetail(length_m=6.64, height_m=2.74, gable_type="Exposed"),
RoomInRoofSurfaceDetail(length_m=6.64, height_m=2.74, gable_type="Party"),
]
assert rir.slopes == [
RoomInRoofSurfaceDetail(length_m=4.94, height_m=1.73),
RoomInRoofSurfaceDetail(length_m=5.0, height_m=2.95),
]
assert rir.common_walls == [
RoomInRoofSurfaceDetail(length_m=4.94, height_m=1.45),
RoomInRoofSurfaceDetail(length_m=4.94, height_m=1.3),
]
assert rir.flat_ceilings == [
RoomInRoofSurfaceDetail(length_m=4.94, height_m=5.17),
]
# The same survey's RIR block in full, INCLUDING the per-surface
# "{surface}: Are you able to provide details of the insulation …?"
# questions (the question text wraps onto a second PDF line; the Yes/No
# sits two tokens after the "{prefix}: Are you able …" line). Verbatim
# from accuracy fixture deal 499617935574 tokens — the dwelling whose
# loft hatch was screwed shut, so every roof-going surface answers "No".
_RIR_BLOCK_WITH_INSULATION_QUESTIONS = [
"Are there rooms in the roof?",
"Yes",
"Roof room age range:",
"B: 1900 - 1929",
"Floor area of room in roof:",
"32.35 m²",
"Are details of the room in roof known?",
"Yes",
"Are details of the gable walls known?",
"Gable wall 1 & 2",
"Gable wall 1 length:",
"6.64 m",
"Gable wall 1 height:",
"2.74 m",
"Page 7",
"",
"Gable wall 1 U-Value:",
"Unknown",
"Gable wall 1: Are you able to provide details of the",
"insulation type?",
"Yes",
"Gable wall 1 type:",
"Exposed",
"Gable wall 2 length:",
"6.64 m",
"Gable wall 2 height:",
"2.74 m",
"Gable wall 2 U-Value:",
"Unknown",
"Gable wall 2: Are you able to provide details of the",
"insulation type?",
"Yes",
"Gable wall 2 type:",
"Party",
"Are details of the stud walls known?",
"None",
"Are details of the slope walls known?",
"Slope 1 & 2",
"Slope 1 length:",
"4.94 m",
"Slope 1 height:",
"1.73 m",
"Slope 1 U-Value:",
"Unknown",
"Slope 1: Are you able to provide details of the",
"insulation type and thickness?",
"No",
"Slope 2 length:",
"5 m",
"Slope 2 height:",
"2.95 m",
"Slope 2 U-Value:",
"Unknown",
"Slope 2: Are you able to provide details of the",
"insulation type and thickness?",
"No",
"Are details of the common walls known?",
"Common wall 1 & 2",
"Common wall 1 length:",
"4.94 m",
"Common wall 1 height:",
"1.45 m",
"Common wall 1 U-Value:",
"Unknown",
"Common wall 2 length:",
"4.94 m",
"Common wall 2 height:",
"1.3 m",
"Common wall 2 U-Value:",
"Unknown",
"Are details of the flat ceiling known?",
"Flat ceiling 1",
"Flat ceiling 1 length:",
"4.94 m",
"Flat ceiling 1 height:",
"5.17 m",
"Flat ceiling 1 U-Value:",
"Unknown",
"Flat ceiling 1: Are you able to provide details of the",
"insulation thickness, type or location?",
"No",
]
def test_per_surface_insulation_known_answers_are_captured(self) -> None:
# Arrange — fixture 1's roof space with the question-bearing RIR
# block spliced in place of its "rooms in the roof? No" lodging.
lines = load_text_fixture()
idx = lines.index("Are there rooms in the roof?")
spliced = (
lines[:idx]
+ self._RIR_BLOCK_WITH_INSULATION_QUESTIONS
+ lines[idx + 2 :]
)
# Act
rir = (
PasHubRdSapSiteNotesExtractor(spliced)
.extract_roof_space()
.main_building.room_in_roof
)
# Assert — gables answered Yes, slopes/flat ceiling answered No;
# common walls are never asked, so stay None (unknown).
assert rir is not None
assert rir.gables is not None
assert [g.insulation_known for g in rir.gables] == [True, True]
assert rir.slopes is not None
assert [s.insulation_known for s in rir.slopes] == [False, False]
assert rir.common_walls is not None
assert [c.insulation_known for c in rir.common_walls] == [None, None]
assert rir.flat_ceilings is not None
assert [f.insulation_known for f in rir.flat_ceilings] == [False]
# The Simplified-approach RIR block ("Are details of the room in roof
# known? No" → "Roof room type: RR type 2"), gables + common walls only,
# no slopes/flat ceilings. Verbatim from accuracy fixture 499538003156
# (78 North Road), including the mid-block "Page 8" break and the noise
# lines the surveyor tool interleaves between a key and its value.
_RIR_SIMPLIFIED_TYPE2_BLOCK = [
"Are there rooms in the roof?",
"Yes",
"Roof room age range:",
"C: 1930 - 1949",
"Floor area of room in roof:",
"25.43 m²",
"Are details of the room in roof known?",
"No",
"Roof room type:",
"RR type 2 (With the Accessible areas of continuous common walls)",
"Gable wall 1 length:",
"5.82 m",
"Gable wall 1 height:",
"2.72 m",
"Gable wall 1 type:",
"Party",
"Gable wall 2 length:",
"5.82 m",
"Gable wall 2 height:",
"2.72 m",
"Gable wall 2 type:",
"Party",
"Common wall 1 length:",
"4.37 m",
"Page 8",
"",
"Common wall 1 height:",
"2.34 m",
"Common wall 2 length:",
"4.37 m",
"Common wall 2 height:",
"2.34 m",
]
def test_roof_room_type_label_is_captured(self) -> None:
# Arrange — fixture 1's roof space with the Simplified RIR block
# spliced in place of its "rooms in the roof? No" lodging.
lines = load_text_fixture()
idx = lines.index("Are there rooms in the roof?")
spliced = lines[:idx] + self._RIR_SIMPLIFIED_TYPE2_BLOCK + lines[idx + 2 :]
# Act
rir = (
PasHubRdSapSiteNotesExtractor(spliced)
.extract_roof_space()
.main_building.room_in_roof
)
# Assert
assert rir is not None
assert (
rir.roof_room_type
== "RR type 2 (With the Accessible areas of continuous common walls)"
)
# Simplified lodgement: gables + common walls, no roof-going surfaces.
assert rir.slopes is None
assert rir.flat_ceilings is None
assert rir.common_walls == [
RoomInRoofSurfaceDetail(length_m=4.37, height_m=2.34),
RoomInRoofSurfaceDetail(length_m=4.37, height_m=2.34),
]
def test_detailed_lodgement_has_no_roof_room_type(self) -> None:
# Arrange — the Detailed §3.10 block (slopes/flat ceilings lodged)
# never carries a "Roof room type:" line.
lines = load_text_fixture()
idx = lines.index("Are there rooms in the roof?")
spliced = lines[:idx] + self._RIR_BLOCK + lines[idx + 2 :]
# Act
rir = (
PasHubRdSapSiteNotesExtractor(spliced)
.extract_roof_space()
.main_building.room_in_roof
)
# Assert
assert rir is not None
assert rir.roof_room_type is None
def test_no_rooms_in_roof_leaves_detail_absent(self) -> None:
# Arrange / Act — fixture 1 as lodged ("No").
detail = PasHubRdSapSiteNotesExtractor(
load_text_fixture()
).extract_roof_space().main_building
# Assert
assert detail.rooms_in_roof is False
assert detail.room_in_roof is None
class TestRoomCountElements:
@pytest.fixture
def rce(self) -> RoomCountElements:

95
backend/documents_parser/tests/test_pashub_sap_accuracy.py
View file

@ -63,21 +63,86 @@ _KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel
_FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy"
_MANIFEST_PATH = _FIXTURES_DIR / "manifest.json"
# --- Ratchets (never loosen), mirroring test_sap_accuracy_corpus.py. ----------
# First real aggregate: the main-heating control-code mapper fix (#1557) is the
# gap that unblocks computation for the cohort. Before it, every fixture raised
# `UnmappedSapCode: main_heating_control` (0/205 computed, aggregate xfailed);
# with it, 201/205 compute (4 still blocked on the residual fuel-code gap,
# e.g. Bulk LPG). Observed on this branch, with the control fix layered on the
# full merged sibling stack to date (fabric #1559-#1562 plus secondary-fuel
# #1564 and roof/country #1566): SAP within-0.5 = 12.4% (25/201), MAE = 2.560.
# The within-0.5 count is one distributional metric that moved *down* a touch
# (13.4% -> 12.4%) as those later fixes landed even though MAE fell (2.65 ->
# 2.56) and is not loosened relative to the current integrated state — it is set
# to what the current codebase actually produces. Ratchet up as later field
# fixes land, per #1568.
_MIN_WITHIN_HALF: float = 0.124
_MAX_SAP_MAE: float = 2.561
# --- Ratchets (never loosen at fixed coverage), mirroring test_sap_accuracy_corpus.py.
# The last integrated state computed 201/205 (4 blocked on residual
# `UnmappedPasHubLabel` gaps) at SAP within-0.5 = 12.4% (25/201), MAE = 2.560.
# Closing those last gaps — secondary fuel 'Mains Gas'/'House Coal' and the
# Table 4e Group 3 heat-network control 'Charging system linked to use of
# community heating, room thermostat only' — brings the cohort to **205/205
# computed (0 blocked)**. That is coverage growth, not a loosening: the
# within-0.5 *count* is unchanged at 25, so no fixture regressed; the fraction
# is 25/205 = 12.2% only because the 4 newly-computable fixtures all sit outside
# 0.5. MAE rises 2.560 -> 2.701 because two of those four are large, isolated
# divergences NOT caused by the mappings (which land the others at d=-1.7/+7.6):
# - 499584755922 d=-33.0 Bulk-LPG main + house-coal secondary (34.0 vs pre_sap 67)
# - 507644414148 d=+18.2 community heating — pashub path maps only the control
# code, not the full heat-network fuel/flags (70.2 vs 52)
# Both are tracked for verification (pre_sap correctness vs a deeper modelling
# gap) — see docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md. Re-baselined to the
# full-coverage aggregate; ratchet back DOWN as those two are resolved.
# Ratcheted 0.12 -> 0.18 / 2.55 -> 1.75 by #1590 bug 4 (mechanical ventilation
# dispatched into the §2 ACH cascade instead of defaulting to NATURAL): 76/205
# cohort fixtures lodge a mechanical system; observed 18.5% within-0.5,
# MAE 1.730 — the campaign's largest single move.
# Ratcheted 0.18 -> 0.19 by the dig-round-3 fixes (see the MAE note below):
# observed 20.0% (41/205).
# Ratcheted 0.19 -> 0.23 by the window-orientation fix (see the MAE note
# below): observed 23.9% (49/205) — the campaign's largest within-0.5 move.
_MIN_WITHIN_HALF: float = 0.23
# Ratcheted 2.71 -> 2.64 by #1590 bug 2 (roof "Insulation At: None" -> explicit
# zero thickness): observed MAE 2.632 — the two ~7-SAP roof over-raters
# (507665533138 / 507670893756) correcting is exactly 14/205 ≈ 0.07 of MAE.
# Ratcheted 2.64 -> 2.58 by #1590 bug 1 (surveyed PV arrays now reach the
# Appendix M input): observed MAE 2.571 — 507639151843's 13.5-SAP zero-PV-credit
# under-rate closing.
# Ratcheted 2.58 -> 2.55 by #1590 bug 3 (PV Connection label -> gov-API int:
# "Not connected" no longer silently credited): observed MAE 2.538.
# Ratcheted 1.75 -> 1.70 by the dig-round-2 fixes: RIR surfaces billed with
# the surveyed roof insulation + stud-wall commons, and community heating
# lodging Table 4a code 301 + Table 12 fuel 51 (507644414148 computes on the
# heat-network branch). Observed MAE 1.684.
# Ratcheted 1.70 -> 1.56 after Khalim manually verified 4 suspect pre_sap
# values in pashub (2026-07-14): 9 Council Houses CW5 8AP is F33 (manifest had
# a stale D67 — our 34.0 was right within a point) and Brightholme M11 4WE is
# the accredited 43 (was pashub's preliminary 53). 58 Hackle Street D58 and
# 16 Bingley Close E52 were CONFIRMED correct — our +7.8/+5.7 deviations there
# are extraction bugs to hunt. Observed MAE 1.551.
# Ratcheted 1.56 -> 1.39 (and within-0.5 0.18 -> 0.19) by the dig-round-3
# fixes: (a) RIR roof-insulation threading now respects the per-surface
# "insulation known? No" answers (Brightholme, whose loft hatch was screwed
# shut, +7.3 -> +0.6); (b) the surveyed "Wall Dry-Lined? Yes" (39/205
# fixtures) reaches the §5.8/Table 14 wall-U adjustment; (c) the Bingley
# community residual — cylinder insulation label int-coded so storage loss
# bills, Table 3 primary loss + RdSAP p.59 flat-rate charging default on the
# water-only heat-network branch (+5.7 -> +3.2). Observed 20.0% / MAE 1.383.
# Held at 1.39 (not loosened) after the secondary-heating-type coding fix
# (raw "Secondary System:" label int-coded to its Table 4a room-heater code
# instead of silently defaulting to code 693 eff 1.00): correctness fix,
# 499607224524 +4.0 -> +1.2, observed MAE 1.383 -> 1.381 — too small to
# tighten the 2-decimal ceiling, within-0.5 unchanged at 20.0%.
# Ratcheted 1.39 -> 1.37 by the Simplified-RIR common-wall fix: a "Roof room
# type" (RdSAP §3.9.2) lodgement now bills common walls as `common_wall` at
# the main-wall U (spec area L x (0.25 + H)) instead of a Detailed `stud_wall`
# roof surface that dropped to the Table 18 col(4) 2.30 uninsulated default
# (78/72 North Road, +1.1 SAP each). Observed MAE 1.3703, within-0.5 20.0%.
# DELIBERATELY LOOSENED 1.371 -> 1.41 by the window-orientation fix (Khalim
# authorised, 2026-07-15) — the ONE sanctioned exception to "ratchets never
# loosen". PasHub lodged `orientation` as a raw string ("South East") where
# the solar-gains cascade only accepts the int octant 1..8, so EVERY window's
# solar gain was silently zeroed (100% of the cohort, ~1,647 windows). Coding
# it (`_pashub_orientation_int`) is an unambiguous correctness fix that moves
# within-0.5 20.0% -> 23.9% (41 -> 49; the campaign's largest within-0.5 move)
# and within-1 87 -> 98. MAE RISES 1.370 -> 1.405 ONLY because the zeroed
# gains were masking the known SAP-10.2-engine-vs-lodged-2012-RdSAP offset the
# module docstring already flags (bias -0.67 -> +1.04) — NOT an extraction
# regression. The correct oracle for this field is the Elmhurst 10.2 worksheet,
# not the hybrid `pre_sap` gauge; validate there before tightening further.
# Nudged 1.41 -> 1.411 by the #1592 revert of the water-only heat-network
# CALCULATOR changes (primary loss + flat-rate charging leaked into the gov-API
# RdSAP corpus). 16 Bingley Close reverts +3.2 -> +4.3, so observed MAE 1.405 ->
# 1.4103; re-tighten when the water-only branch is fixed correctly under #1592
# (the cylinder-insulation MAPPER fix is retained, so the revert is partial).
_MAX_SAP_MAE: float = 1.411
_KNOWN_GAP_REASON = (
"pashub `from_site_notes` mapper does not int-code a main-heating "

2
datatypes/epc/domain/epc_property_data.py
View file

@ -306,7 +306,7 @@ class SapConservatory:
class SapWindow:
frame_material: Optional[str]
glazing_gap: Union[int, str]
orientation: Union[int, str]
orientation: Optional[Union[int, str]] # SAP10 octant 1..8; None → no solar gain
window_type: Union[int, str]
glazing_type: Union[int, str]
window_width: float

464
datatypes/epc/domain/mapper.py
View file

@ -6,6 +6,7 @@ from datetime import date
from decimal import ROUND_HALF_UP, Decimal
from typing import (
Any,
Callable,
Dict,
Final,
List,
@ -153,6 +154,8 @@ from datatypes.epc.surveys.pashub_rdsap_site_notes import (
FloorMeasurement,
HeatingAndHotWater,
PasHubRdSapSiteNotes,
Renewables as PasHubRenewables,
RoomInRoofSurfaceDetail,
RoofSpaceDetail,
Ventilation,
WaterUse,
@ -366,7 +369,8 @@ class EpcPropertyDataMapper:
is_dwelling_export_capable=general.dwelling_export_capable,
wind_turbines_terrain_type=general.terrain_type,
electricity_smart_meter_present=general.electricity_smart_meter,
pv_connection=renewables.pv_connection,
pv_connection=_pashub_pv_connection_code(renewables.pv_connection),
photovoltaic_arrays=_pashub_pv_arrays(renewables),
photovoltaic_supply=(
PhotovoltaicSupply(
none_or_no_details=PhotovoltaicSupplyNoneOrNoDetails(
@ -5961,6 +5965,14 @@ def _map_roof(
) -> tuple[Optional[str], Optional[Union[str, int]]]:
if roof is None:
return None, None
# "Insulation At: None" is the assessor explicitly recording ZERO loft
# insulation (no thickness line follows — there is nothing to measure).
# Lodge an explicit 0 so `u_roof` takes the Table 16 uninsulated row
# (2.30 W/m²K) instead of falling through, on thickness=None, to the
# age-band "assumed insulated" default (issue #1590 bug 2). A blank
# `insulation_at` stays None — "not surveyed", not "no insulation".
if roof.insulation_at == "None":
return roof.insulation_at, 0
thickness: Optional[Union[str, int]] = (
roof.insulation_thickness_mm
if roof.insulation_thickness_mm is not None
@ -5969,6 +5981,108 @@ def _map_roof(
return roof.insulation_at or None, thickness
# PasHub gable "type" label → the Table 4 surface kind the cascade's
# per-surface RR branch dispatches on (Exposed = as-common-wall U,
# Party = 0.25 W/m²K).
_PASHUB_GABLE_TYPE_TO_KIND: Dict[str, str] = {
"Exposed": "gable_wall_external",
"Party": "gable_wall",
}
def _pashub_room_in_roof(
roof: Union[RoofSpaceDetail, ExtensionRoofSpace],
) -> Optional[SapRoomInRoof]:
"""Build `SapRoomInRoof` from the surveyed §Roof-Space room-in-roof block
(RdSAP §3.10 detailed measurement), mirroring `_api_build_room_in_roof`:
every surveyed surface (gable / slope / common wall / flat ceiling)
becomes a `detailed_surfaces` entry of length × height, so the cascade
bills exact RIR surfaces instead of treating the whole shell as
well-insulated loft (issue #1590 bug 5). Returns None when no RIR is
lodged. A gable with an unmapped type label strict-raises."""
rir = roof.room_in_roof
if not roof.rooms_in_roof or rir is None:
return None
surfaces: List[SapRoomInRoofSurface] = []
# PasHub lodges roof insulation ONCE at roof-space level, not per RIR
# surface. Thread it into the roof-going surfaces (slope / stud_wall /
# flat_ceiling — Table 17) so they aren't billed at the Table 18 col(4)
# "unknown" full-uninsulated default while the same survey records an
# insulated roof. Gables keep None — they are walls, priced by wall
# fields, and their consumer branches ignore the thickness anyway.
roof_thickness = (
roof.insulation_thickness_mm
if isinstance(roof.insulation_thickness_mm, int)
else None
)
# Newer survey forms ask per surface "Are you able to provide details of
# the insulation …?" — a "No" (fixture 499617935574: loft hatch screwed
# shut) means the roof-space thickness is NOT evidence for that surface,
# which must fall back to the Table 18 col(4) age-band "unknown" default.
# On such a form, surfaces the question never covers (common walls) get
# no threading either. Legacy forms (question absent everywhere) keep the
# unconditional threading above.
form_asks_insulation_known = any(
d.insulation_known is not None
for group in (rir.gables, rir.slopes, rir.common_walls, rir.flat_ceilings)
for d in group or []
)
def _add(
details: Optional[List[RoomInRoofSurfaceDetail]],
kind: Optional[str],
insulation_thickness_mm: Optional[int] = None,
area_of: Callable[[float, float], float] = lambda length, height: length
* height,
) -> None:
for d in details or []:
if d.length_m is None or d.height_m is None:
continue
surface_kind = kind
if surface_kind is None: # gable — kind from the surveyed type
if d.gable_type not in _PASHUB_GABLE_TYPE_TO_KIND:
raise UnmappedPasHubLabel(
"room-in-roof gable type", str(d.gable_type)
)
surface_kind = _PASHUB_GABLE_TYPE_TO_KIND[d.gable_type]
thickness = insulation_thickness_mm
if form_asks_insulation_known and not d.insulation_known:
thickness = None
surfaces.append(
SapRoomInRoofSurface(
kind=surface_kind,
area_m2=area_of(d.length_m, d.height_m),
u_value=d.u_value,
insulation_thickness_mm=thickness,
)
)
_add(rir.gables, None)
_add(rir.slopes, "slope", roof_thickness)
if rir.roof_room_type is not None:
# Simplified approach ("Roof room type" lodged, RdSAP §3.9.2): a
# "common wall" is billed as an external wall at the storey-below
# main-wall U (SAP kind `common_wall`), spec area L × (0.25 + H) —
# NOT a Detailed §3.10 `stud_wall` roof surface, which would drop to
# the Table 18 col(4) "all elements uninsulated" default on a no-
# loft-access dwelling (78/72 North Road, deals 499538003156/
# 499627236560).
_add(rir.common_walls, "common_wall", area_of=lambda l, h: l * (0.25 + h))
else:
# Detailed §3.10: "common wall" lodges as a stud wall (Table 17 col
# 3, raw L × H area); every worksheet-validated Detailed fixture
# (000516/000487/000477/000480) uses stud_wall for this low wall.
_add(rir.common_walls, "stud_wall", roof_thickness)
_add(rir.flat_ceilings, "flat_ceiling", roof_thickness)
if rir.floor_area_m2 is None:
return None
return SapRoomInRoof(
floor_area=rir.floor_area_m2,
construction_age_band=_extract_age_band(rir.age_range or ""),
detailed_surfaces=surfaces or None,
)
def _map_main_building_part(
construction: BuildingConstruction,
measurements: BuildingMeasurements,
@ -5979,7 +6093,13 @@ def _map_main_building_part(
roof_location, roof_thickness = _map_roof(roof)
return SapBuildingPart(
identifier=BuildingPartIdentifier.MAIN,
sap_room_in_roof=_pashub_room_in_roof(roof),
construction_age_band=_extract_age_band(main.age_range),
# PasHub surveys never lodge a basement wall; pin the flag so a
# "System Build" wall (code 6, which doubles as the gov-API basement
# sentinel) is not silently routed through the basement wall/floor
# cascade (issue #1590 bug 7).
wall_is_basement=False,
wall_construction=_pashub_wall_construction_int(
main.walls_construction_type
),
@ -5987,6 +6107,9 @@ def _map_main_building_part(
main.walls_insulation_type
),
wall_thickness_measured=main.wall_thickness_mm is not None,
# "Wall Dry-Lined? Yes" routes the wall U through the RdSAP10 §5.8 /
# Table 14 adjustment; None (not asked) leaves the cascade default.
wall_dry_lined=main.dry_lined,
party_wall_construction=_pashub_party_wall_construction_int(
main.party_wall_construction_type
),
@ -6010,7 +6133,13 @@ def _map_extension_building_part(
roof_location, roof_thickness = _map_roof(roof)
return SapBuildingPart(
identifier=BuildingPartIdentifier.extension(ext_c.id),
sap_room_in_roof=_pashub_room_in_roof(roof) if roof is not None else None,
construction_age_band=_extract_age_band(ext_c.age_range),
# PasHub surveys never lodge a basement wall; pin the flag so a
# "System Build" wall (code 6, which doubles as the gov-API basement
# sentinel) is not silently routed through the basement wall/floor
# cascade (issue #1590 bug 7).
wall_is_basement=False,
wall_construction=_pashub_wall_construction_int(
ext_c.walls_construction_type
),
@ -6018,6 +6147,9 @@ def _map_extension_building_part(
ext_c.walls_insulation_type
),
wall_thickness_measured=ext_c.wall_thickness_mm is not None,
# "Wall Dry-Lined? Yes" routes the wall U through the RdSAP10 §5.8 /
# Table 14 adjustment; None (not asked) leaves the cascade default.
wall_dry_lined=ext_c.dry_lined,
party_wall_construction=_pashub_party_wall_construction_int(
ext_c.party_wall_construction_type
),
@ -6035,7 +6167,7 @@ def _map_sap_window(window: Window) -> SapWindow:
return SapWindow(
frame_material=window.frame_type,
glazing_gap=window.glazing_gap,
orientation=window.orientation,
orientation=_pashub_orientation_int(window.orientation),
window_type=window.window_type,
glazing_type=_pashub_glazing_type_int(window.glazing_type),
window_width=window.width_m,
@ -6091,12 +6223,26 @@ def _map_sap_heating(
else None
)
# Heat networks lodge the Table 4a code + Table 12 community fuel from
# the community heat-source label; every other system resolves the plain
# Table 32 fuel (issue #1590 follow-up, fixture 507644414148).
community_sap_code: Optional[int] = None
main_fuel: Union[int, str]
if main.system_type.lower() in _PASHUB_HEAT_NETWORK_SYSTEM_TYPES:
community_sap_code, community_fuel = _pashub_community_heating(
main.community_heat_source, fuel_type
)
main_fuel = community_fuel if community_fuel is not None else fuel_type
else:
main_fuel = _pashub_main_fuel_code(fuel_type)
return SapHeating(
instantaneous_wwhrs=InstantaneousWwhrs(),
main_heating_details=[
MainHeatingDetail(
has_fghrs=main.flue_gas_heat_recovery_system,
main_fuel_type=_pashub_main_fuel_code(fuel_type),
main_fuel_type=main_fuel,
sap_main_heating_code=community_sap_code,
heat_emitter_type=_pashub_heat_emitter_code(main.emitter),
emitter_temperature=main.emitter_temperature,
fan_flue_present=main.fan_assist,
@ -6115,14 +6261,21 @@ def _map_sap_heating(
],
has_fixed_air_conditioning=ventilation.has_fixed_air_conditioning,
secondary_fuel_type=secondary_fuel_type,
secondary_heating_type=heating.secondary_heating.secondary_system,
secondary_heating_type=_pashub_secondary_heating_type_code(
heating.secondary_heating.secondary_system
),
shower_outlets=shower_outlets,
cylinder_size=(
heating.water_heating.cylinder_size
_pashub_cylinder_size_code(
heating.water_heating.cylinder_size,
heating.water_heating.system,
)
if cylinder_present
else None
),
cylinder_insulation_type=heating.water_heating.insulation_type,
cylinder_insulation_type=_pashub_cylinder_insulation_type_code(
heating.water_heating.insulation_type, cylinder_present
),
cylinder_insulation_thickness_mm=heating.water_heating.insulation_thickness_mm,
water_heating_code=water_heating_code,
water_heating_fuel=water_heating_fuel,
@ -6132,9 +6285,168 @@ def _map_sap_heating(
)
# PasHub surveyed "Ventilation type" label → `MechanicalVentilationKind` enum
# name for the §2 cascade dispatch, mirroring the gov-API table
# `_API_MECHANICAL_VENTILATION_TO_KIND` (dMEV=3 / MEV=2 / PIV-outside=6 →
# EXTRACT_OR_PIV_OUTSIDE; PIV-from-loft=5 is "as natural" → None). Left None,
# `ventilation_from_cert` silently defaults a mechanical system to NATURAL
# (24d), dropping its ACH heat loss (issue #1590 bug 4).
_PASHUB_VENTILATION_TYPE_TO_KIND: Dict[str, Optional[str]] = {
"Natural": None,
"Mechanical Extract - Decentralised": "EXTRACT_OR_PIV_OUTSIDE",
"Mechanical Extract - Centralised": "EXTRACT_OR_PIV_OUTSIDE",
"Positive Input - From Outside": "EXTRACT_OR_PIV_OUTSIDE",
"Positive Input - From Loft (As Natural)": None,
}
def _pashub_mechanical_ventilation_kind(
ventilation_type: Optional[str],
) -> Optional[str]:
"""Resolve a PasHub surveyed Ventilation type label to the
`MechanicalVentilationKind` enum name at the mapper boundary (ADR-0015).
Blank stays None (natural); a non-empty label the lookup does not cover
strict-raises `UnmappedPasHubLabel` so the gap is fixed here, never
silently under-stating ventilation heat loss downstream."""
if not ventilation_type:
return None
if ventilation_type not in _PASHUB_VENTILATION_TYPE_TO_KIND:
raise UnmappedPasHubLabel("ventilation type", ventilation_type)
return _PASHUB_VENTILATION_TYPE_TO_KIND[ventilation_type]
# PasHub community heat-source label ("Heating System (Other)") → the
# Table 4a heat-network SAP code that routes `is_heat_network_main` (DLF,
# standing charge, 80% heat-source efficiency) and, via the shared Elmhurst
# resolver, the Table 12 community fuel. Only the cohort's boilers-only
# variant is lodged so far; CHP (302) / heat pump (304) join when observed.
_PASHUB_COMMUNITY_HEAT_SOURCE_TO_SAP_CODE: Dict[str, int] = {
"Community heating - boilers only": 301,
}
_PASHUB_COMMUNITY_HEAT_SOURCE_TO_ELMHURST: Dict[str, str] = {
"Community heating - boilers only": "Boilers",
}
def _pashub_community_heating(
heat_source_label: str, fuel_label: str
) -> tuple[int, Optional[int]]:
"""Resolve a PasHub community-heating lodgement to its (Table 4a SAP
code, Table 12 fuel code) at the mapper boundary (ADR-0015), reusing
`_resolve_community_heating_fuel_code`. An uncovered heat-source label
strict-raises so the dwelling is never silently priced as an ordinary
boiler (issue #1590 follow-up, fixture 507644414148)."""
if heat_source_label not in _PASHUB_COMMUNITY_HEAT_SOURCE_TO_SAP_CODE:
raise UnmappedPasHubLabel("community heat source", heat_source_label)
sap_code = _PASHUB_COMMUNITY_HEAT_SOURCE_TO_SAP_CODE[heat_source_label]
fuel = _resolve_community_heating_fuel_code(
_PASHUB_COMMUNITY_HEAT_SOURCE_TO_ELMHURST[heat_source_label],
fuel_label,
)
return sap_code, fuel
# PasHub surveyed §Water-Heating "Cylinder Size" band → SAP10 cylinder-size
# enum (Table 28: 2=Normal/110L, 3=Medium/160L, 4=Large/210L). PasHub renders
# the band with its own litre-range suffix (unlike the Elmhurst forms in
# `_ELMHURST_CYLINDER_SIZE_LABEL_TO_SAP10`); the raw string was int-or-noned by
# `_cylinder_volume_l_from_code`, silently skipping the Table 28 volume
# convention (issue #1590 bug 6).
_PASHUB_CYLINDER_SIZE_TO_SAP10: Dict[str, int] = {
"Normal (90-130 litres)": 2,
"Large (>170 litres)": 4,
}
def _pashub_cylinder_size_code(
size_label: Optional[str], system_label: str
) -> Optional[int]:
"""Resolve a PasHub surveyed Cylinder Size band to the SAP10 enum at the
mapper boundary (ADR-0015). "No Access" follows RdSAP 10 Table 28 (p.55):
off-peak dual electric immersion → 210 L, solid-fuel boiler → 160 L,
otherwise → 110 L (code 2) — the cohort's case is a cylinder fed from the
gas main. An electric-immersion "No Access" needs the meter-type context
this path doesn't yet plumb, so it strict-raises (loud gap, mirroring the
cohort-scoped control approach) rather than silently taking the 110 L
branch; any other uncovered non-empty label strict-raises too."""
if not size_label or size_label == "No Cylinder":
return None
if size_label == "No Access":
if system_label == "Electric immersion":
raise UnmappedPasHubLabel(
"cylinder size",
"'No Access' on an electric immersion needs meter-type "
"context (RdSAP 10 Table 28 210L branch) — not yet mapped",
)
return 2 # Table 28 "otherwise": 110 L
code = _PASHUB_CYLINDER_SIZE_TO_SAP10.get(size_label)
if code is None:
raise UnmappedPasHubLabel("cylinder size", size_label)
return code
# PasHub surveyed "Secondary System:" label → SAP 10.2 Table 4a room-heater
# code (PCDB `room_heaters` descriptions). Passed through raw, the string
# hits `_secondary_efficiency`'s `_int_or_none` and silently defaults to
# code 693 (portable electric heater, eff 1.00) — over-crediting a gas/solid
# open fire that should bill at 40-63% / 32%.
_PASHUB_SECONDARY_HEATING_TYPE_TO_SAP10: Dict[str, int] = {
# Electric direct-acting (eff 1.00) — modal cohort heater.
"Panel, convector or radiant heaters": 691,
# Gas room heaters (Table 4a column A).
"Flush fitting live fuel effect open front fire, sealed to fireplace": 605,
"1980 or later - Open front fire with open flue, sealed to fireplace": 603,
# Solid fuel (eff 0.32 column B).
"Open fire in grate": 631,
}
def _pashub_secondary_heating_type_code(
system_label: Optional[str],
) -> Optional[int]:
"""Resolve a PasHub surveyed secondary-heating system label to its SAP
10.2 Table 4a code at the mapper boundary (ADR-0015). Blank/"No
Secondary Heating" → None; any uncovered non-empty label strict-raises
(mirroring `_pashub_secondary_fuel_code`)."""
if not system_label or system_label == "No Secondary Heating":
return None
code = _PASHUB_SECONDARY_HEATING_TYPE_TO_SAP10.get(system_label)
if code is None:
raise UnmappedPasHubLabel("secondary heating type", system_label)
return code
# PasHub surveyed §Water-Heating cylinder "Insulation Type:" label → SAP10
# `cylinder_insulation_type` cascade code (foam=1 / jacket=2, mirroring
# `_ELMHURST_CYLINDER_INSULATION_LABEL_TO_SAP10`). Passed through raw, the
# storage-loss resolver's `_int_or_none` yields None and a real lodged
# cylinder is billed ZERO storage loss.
_PASHUB_CYLINDER_INSULATION_TO_SAP10: Dict[str, int] = {
"Factory fitted": 1, # factory-applied foam
}
def _pashub_cylinder_insulation_type_code(
insulation_label: Optional[str], cylinder_present: bool
) -> Optional[int]:
"""Resolve a PasHub surveyed cylinder insulation label to the SAP10 code
at the mapper boundary (ADR-0015). Returns None when no cylinder is
present or the label is genuinely absent; any uncovered non-empty label
strict-raises (mirroring `_pashub_cylinder_size_code`)."""
if not cylinder_present or not insulation_label:
return None
code = _PASHUB_CYLINDER_INSULATION_TO_SAP10.get(insulation_label)
if code is None:
raise UnmappedPasHubLabel("cylinder insulation", insulation_label)
return code
def _map_sap_ventilation(ventilation: Ventilation) -> SapVentilation:
return SapVentilation(
ventilation_type=ventilation.ventilation_type,
mechanical_ventilation_kind=_pashub_mechanical_ventilation_kind(
ventilation.ventilation_type
),
draught_lobby=ventilation.draught_lobby,
pressure_test=ventilation.pressure_test,
open_flues_count=ventilation.number_of_open_flues,
@ -6689,6 +7001,63 @@ _ELMHURST_PV_OVERSHADING_TO_RDSAP: Dict[str, int] = {
}
# PasHub surveyed "PV Connection" label → the gov-API int enum the calculator's
# Appendix M credit gate keys off (`_pv_connected_to_dwelling_meter`): 1 = PV
# present but NOT connected to the dwelling's meter (zero credit — communal /
# separately-metered), 2 = connected (credited). The gate treats any non-int as
# "connected", so an unresolved label silently credits PV the dwelling doesn't
# get to count (issue #1590 bug 3).
_PASHUB_PV_CONNECTION_TO_GOV_API: Dict[str, int] = {
"Not connected to electricity meter": 1,
"Connected to dwellings electricity meter": 2,
}
def _pashub_pv_connection_code(connection_label: Optional[str]) -> Optional[int]:
"""Resolve a PasHub surveyed PV Connection label to the gov-API int enum at
the mapper boundary (ADR-0015). Absent stays None (no PV lodged); a
non-empty label the lookup does not cover strict-raises
`UnmappedPasHubLabel` so the gap is fixed here, never silently credited
downstream."""
if not connection_label:
return None
code = _PASHUB_PV_CONNECTION_TO_GOV_API.get(connection_label)
if code is None:
raise UnmappedPasHubLabel("PV connection", connection_label)
return code
def _pashub_pv_arrays(
renewables: PasHubRenewables,
) -> Optional[List[PhotovoltaicArray]]:
"""Build the Appendix M cascade's input list from the PasHub Renewables
per-system PV block, mirroring `_elmhurst_pv_arrays` below and reusing its
format-agnostic converters. PasHub lodges the orientation space-separated
("South East") where the shared octant map keys hyphenated ("South-East"),
so normalise before the lookup. Returns None when no per-array detail is
lodged — the PV-absent path the cascade already handles (issue #1590
bug 1: dropping this detail silently zeroed the PV credit)."""
if not renewables.pv_arrays:
return None
out: List[PhotovoltaicArray] = []
for arr in renewables.pv_arrays:
if arr.kwp is None or arr.kwp <= 0.0:
continue
out.append(
PhotovoltaicArray(
peak_power=arr.kwp,
pitch=_elmhurst_pv_pitch_code(
arr.pitch_degrees if arr.pitch_degrees is not None else 30
),
orientation=_elmhurst_orientation_int(
(arr.orientation or "").replace(" ", "-")
),
overshading=_elmhurst_pv_overshading_int(arr.overshading),
)
)
return out or None
def _elmhurst_pv_arrays(
renewables: ElmhurstRenewables,
) -> Optional[List[PhotovoltaicArray]]:
@ -6784,6 +7153,23 @@ def _elmhurst_orientation_int(orientation: str) -> int:
return _ELMHURST_ORIENTATION_TO_SAP10.get(orientation, 1)
def _pashub_orientation_int(orientation: Optional[str]) -> Optional[int]:
"""Map a PasHub window orientation label to the SAP10 octant code (1..8)
at the mapper boundary (ADR-0015). PasHub lodges it space-separated
("South East") where the shared octant map keys hyphenated, so normalise
before lookup (mirroring `_pashub_pv_arrays`). Blank → None (the solar-
gains cascade skips a window with no orientation). Any uncovered non-empty
label strict-raises (mirroring the other `_pashub_*` coders) — passed
through raw, the string reaches the int-only `_orientation` and silently
zeroes the window's solar-gain contribution."""
if not orientation:
return None
code = _ELMHURST_ORIENTATION_TO_SAP10.get(orientation.replace(" ", "-"))
if code is None:
raise UnmappedPasHubLabel("window orientation", orientation)
return code
# RdSAP 10 §3.7.1 (PDF p.21) — the source RdSAP data set classifies
# each opening as "Window (vertical)" or "Roof window (inclined)" per
# the assessor's discrete lodgement. The Elmhurst Summary PDF §11.0
@ -7165,6 +7551,14 @@ _PASHUB_MAIN_FUEL_TO_SAP10: Dict[str, int] = {
# not a fuel code.
_PASHUB_SECONDARY_FUEL_TO_SAP10: Dict[str, int] = {
"Electricity": 30, # standard electricity, Table 32 code 30
# Mains-gas secondary heater (e.g. a gas fire): same fuel code the main-fuel
# map pins for mains gas (26 — see `_PASHUB_MAIN_FUEL_TO_SAP10`). Both label
# cases occur in the cohort's site notes, mirroring the main-fuel map.
"Mains gas": 26,
"Mains Gas": 26,
# Solid-fuel secondary heater burning house coal → Table 32 code 11 (House
# coal), the same code the solid-fuel main-heating path pins (SAP 631 → 11).
"House Coal": 11,
}
@ -7229,6 +7623,30 @@ _PASHUB_MAIN_HEATING_CONTROL_TO_SAP10: Dict[str, int] = {
}
# PasHub `system_type` labels that are SAP 10.2 Table 4e Group 3 heat networks —
# the only non-boiler group the cohort surveys. Their control codes live in a
# separate map (below) because the same control label carries a different 2xxx
# code per group (see `_PASHUB_BOILER_SYSTEM_TYPES`).
_PASHUB_HEAT_NETWORK_SYSTEM_TYPES: frozenset[str] = frozenset({
"community heating system",
})
# PasHub Site-Notes main-heating "Controls" labels for a Table 4e Group 3 heat
# network → their SAP code. All four "charging system linked to use of community
# heating" codes (2306/2310/2312/2314) are calculation-identical — control_type
# 3, DLF 1.00/1.00, temperature_adjustment 0.0 (`cert_to_inputs.py`
# `_CONTROL_TYPE_BY_CODE` / DLF / temp-adjustment maps) — so the cohort's
# "...room thermostat only" resolves to 2306, the domain's modal heat-network
# control code (`domain/epc/property_overlays/main_heating_system_overlay.py`,
# `HANDOVER_POST_S0380_176`). Every code here is in `_CONTROL_TYPE_BY_CODE`.
_PASHUB_HEAT_NETWORK_CONTROL_TO_SAP10: Dict[str, int] = {
"Charging system linked to use of community heating, room thermostat only": (
2306
),
}
def _pashub_main_heating_control_code(
control_label: str, system_type: str
) -> Union[int, str]:
@ -7236,24 +7654,34 @@ def _pashub_main_heating_control_code(
Table 4e code at the mapper boundary (ADR-0015). A genuinely blank label is
the "no controls lodged" shape and passes through unchanged.
The resolution is system-aware: the lookup holds only Group 1 (boiler)
codes, and the same control label recurs across Table 4e's system groups
under different 2xxx codes, so a non-boiler `system_type` strict-raises
(naming the system) rather than silently taking a boiler code. A boiler
system whose label the lookup does not cover likewise strict-raises — either
way the gap is fixed here, never mis-rated or strict-raised deep in the
calculator as `UnmappedSapCode`."""
The resolution is system-aware: the same control label recurs across Table
4e's system groups under different 2xxx codes, so the code is chosen by the
system's group — Group 1 boiler (`_PASHUB_MAIN_HEATING_CONTROL_TO_SAP10`) or
Group 3 heat network (`_PASHUB_HEAT_NETWORK_CONTROL_TO_SAP10`). Any other
system group, or a label the group's lookup does not cover, strict-raises
`UnmappedPasHubLabel` (naming the label and system) rather than silently
taking a wrong-group code — the gap is fixed here, never mis-rated or
strict-raised deep in the calculator as `UnmappedSapCode`."""
if not control_label:
return control_label
if system_type.lower() not in _PASHUB_BOILER_SYSTEM_TYPES:
system = system_type.lower()
if system in _PASHUB_BOILER_SYSTEM_TYPES:
group_map = _PASHUB_MAIN_HEATING_CONTROL_TO_SAP10
elif system in _PASHUB_HEAT_NETWORK_SYSTEM_TYPES:
group_map = _PASHUB_HEAT_NETWORK_CONTROL_TO_SAP10
else:
raise UnmappedPasHubLabel(
"main heating control",
f"{control_label!r} on non-boiler system {system_type!r} "
f"(only Table 4e Group 1 boiler codes are mapped)",
f"{control_label!r} on unsupported system {system_type!r} "
f"(only Table 4e Group 1 boiler + Group 3 heat-network codes "
f"are mapped)",
)
code = _PASHUB_MAIN_HEATING_CONTROL_TO_SAP10.get(control_label)
code = group_map.get(control_label)
if code is None:
raise UnmappedPasHubLabel("main heating control", control_label)
raise UnmappedPasHubLabel(
"main heating control",
f"{control_label!r} on system {system_type!r}",
)
return code

819
datatypes/epc/domain/tests/test_from_site_notes.py
View file

@ -74,6 +74,686 @@ class TestRoofConstructionTypeCoding:
assert extension_part.roof_construction_type == "Flat"
class TestPhotovoltaicArrayMapping:
"""A surveyed per-system PV block ("PV 1: kWp value: 2.43 kWp", pitch 30°,
orientation "South East", overshading "None or very little") must reach
`sap_energy_source.photovoltaic_arrays` — the ONLY field the calculator's
Appendix M generation path reads. Dropping it silently zeroes the PV credit
(issue #1590 bug 1; fixture 507639151843 under-rates by 13.6 SAP).
"""
def test_surveyed_pv_array_reaches_photovoltaic_arrays(self) -> None:
# Arrange — the survey's PV block: 2.43 kWp, 30°, SE, no overshading.
data = load("pashub_rdsap_site_notes_example1.json")
data["renewables"]["photovoltaic_array"] = True
data["renewables"]["pv_arrays"] = [
{
"kwp": 2.43,
"pitch_degrees": 30,
"orientation": "South East",
"overshading": "None or very little",
}
]
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert — SAP codes: pitch 30° → 2, "South East" → octant 4,
# "None or very little" → 1 (ZPV=1.0).
arrays = result.sap_energy_source.photovoltaic_arrays
assert arrays is not None
assert len(arrays) == 1
assert arrays[0].peak_power == 2.43
assert arrays[0].pitch == 2
assert arrays[0].orientation == 4
assert arrays[0].overshading == 1
def test_no_pv_block_keeps_arrays_absent(self) -> None:
# Arrange — the fixture as lodged (no PV).
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_energy_source.photovoltaic_arrays is None
class TestCommunityHeatingMapping:
"""A "Community heating system" dwelling must lodge the Table 4a
heat-network SAP code and the Table 12 community fuel — with
`sap_main_heating_code=None`, `is_heat_network_main()` is False and the
calculator's whole heat-network branch (DLF, £120 standing charge, 80%
heat-source efficiency, Table 4c(3) charging) never fires; the dwelling is
priced as an ordinary mains-gas boiler (issue #1590 follow-up; fixture
507644414148 over-rates +14.3, −8.6 of it from these two fields).
"""
def test_community_boilers_lodge_heat_network_code_and_fuel(self) -> None:
# Arrange — the cohort's community lodgement (verbatim survey labels).
data = load("pashub_rdsap_site_notes_example1.json")
mh = data["heating_and_hot_water"]["main_heating"]
mh["system_type"] = "Community heating system"
mh["community_heat_source"] = "Community heating - boilers only"
mh["fuel"] = "Mains Gas"
mh["controls"] = (
"Charging system linked to use of community heating, room thermostat only"
)
mh["product_id"] = 0 # heat networks lodge no PCDB boiler
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
main = EpcPropertyDataMapper.from_site_notes(
survey
).sap_heating.main_heating_details[0]
# Assert — Table 4a code 301 (community boilers) routes
# is_heat_network_main; Table 12 code 51 (community boilers,
# mains gas) prices/factors the heat correctly.
assert main.sap_main_heating_code == 301
assert main.main_fuel_type == 51
def test_unknown_heat_source_strict_raises(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
mh = data["heating_and_hot_water"]["main_heating"]
mh["system_type"] = "Community heating system"
mh["community_heat_source"] = "Some novel scheme"
mh["controls"] = ""
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act / Assert
with pytest.raises(UnmappedPasHubLabel, match="community heat source"):
EpcPropertyDataMapper.from_site_notes(survey)
class TestRoomInRoofMapping:
"""A surveyed room-in-roof block (RdSAP §3.10 detailed measurement) must
reach `sap_building_parts[*].sap_room_in_roof` — left None, ~32 m² of
poorly-insulated RIR surfaces are billed as well-insulated loft (issue
#1590 bug 5; fixture 499617935574 over-rates by ~2-3.5 SAP). Values below
are that survey's verbatim lodgement. Gable types code per Table 4:
Exposed → gable_wall_external, Party → gable_wall.
"""
_RIR = {
"age_range": "B: 1900 - 1929",
"floor_area_m2": 32.35,
"gables": [
{"length_m": 6.64, "height_m": 2.74, "gable_type": "Exposed"},
{"length_m": 6.64, "height_m": 2.74, "gable_type": "Party"},
],
"slopes": [
{"length_m": 4.94, "height_m": 1.73},
{"length_m": 5.0, "height_m": 2.95},
],
"common_walls": [
{"length_m": 4.94, "height_m": 1.45},
{"length_m": 4.94, "height_m": 1.3},
],
"flat_ceilings": [{"length_m": 4.94, "height_m": 5.17}],
}
def test_room_in_roof_reaches_building_part(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["rooms_in_roof"] = True
data["roof_space"]["main_building"]["room_in_roof"] = self._RIR
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
rir = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[
0
].sap_room_in_roof
# Assert
assert rir is not None
assert rir.floor_area == 32.35
assert rir.construction_age_band == "B"
surfaces = rir.detailed_surfaces
assert surfaces is not None
# Common walls lodge as Detailed-mode stud walls (Table 17 col 3) —
# "common_wall" is a §3.9.2 Simplified-Type-2-only concept; every
# worksheet-validated Detailed fixture uses stud_wall for this low
# internal wall.
kinds_areas = [(s.kind, round(s.area_m2, 2)) for s in surfaces]
assert kinds_areas == [
("gable_wall_external", round(6.64 * 2.74, 2)),
("gable_wall", round(6.64 * 2.74, 2)),
("slope", round(4.94 * 1.73, 2)),
("slope", round(5.0 * 2.95, 2)),
("stud_wall", round(4.94 * 1.45, 2)),
("stud_wall", round(4.94 * 1.3, 2)),
("flat_ceiling", round(4.94 * 5.17, 2)),
]
# PasHub lodges roof insulation once at roof-space level (the fixture's
# "Joists"/100mm) — thread it into the roof-going surfaces so they
# aren't billed at the Table 18 "unknown" full-uninsulated default
# while the same survey says the roof is insulated. Gables keep None
# (walls — insulation described by the wall fields, not the loft).
by_kind = {s.kind: s.insulation_thickness_mm for s in surfaces}
assert by_kind["slope"] == 100
assert by_kind["stud_wall"] == 100
assert by_kind["flat_ceiling"] == 100
assert by_kind["gable_wall"] is None
# A Simplified-approach lodgement ("Roof room type: RR type 2", details
# NOT known): gables + common walls only, no slopes/flat ceilings.
# Verbatim from fixture 499538003156 (78 North Road).
_RIR_SIMPLIFIED = {
"age_range": "C: 1930 - 1949",
"floor_area_m2": 25.43,
"roof_room_type": "RR type 2 (With the Accessible areas of continuous common walls)",
"gables": [
{"length_m": 5.82, "height_m": 2.72, "gable_type": "Party"},
{"length_m": 5.82, "height_m": 2.72, "gable_type": "Party"},
],
"common_walls": [
{"length_m": 4.37, "height_m": 2.34},
{"length_m": 4.37, "height_m": 2.34},
],
}
def test_simplified_common_walls_bill_as_common_wall(self) -> None:
"""On the Simplified approach (a "Roof room type" is lodged), a
"common wall" is a §3.9.2 Type-2 concept billed as an external wall
at the main-wall U — SAP kind `common_wall`, spec area L × (0.25 + H)
(RdSAP 10 §3.9.2 / heat_transmission's Simplified-BP area rule) — NOT
a Detailed §3.10 `stud_wall` roof surface, which would drop to the
Table 18 col(4) "all elements uninsulated" default (2.30 W/m²K) over
~20 m² on the no-loft-access North Road pair (deals
499538003156/499627236560, over-rate ~5 SAP before this fix)."""
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["rooms_in_roof"] = True
data["roof_space"]["main_building"]["room_in_roof"] = self._RIR_SIMPLIFIED
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
rir = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[
0
].sap_room_in_roof
# Assert
assert rir is not None
surfaces = rir.detailed_surfaces
assert surfaces is not None
cw_area = round(4.37 * (0.25 + 2.34), 2) # L × (0.25 + H)
kinds_areas = [(s.kind, round(s.area_m2, 2)) for s in surfaces]
assert kinds_areas == [
("gable_wall", round(5.82 * 2.72, 2)),
("gable_wall", round(5.82 * 2.72, 2)),
("common_wall", cw_area),
("common_wall", cw_area),
]
def test_detailed_common_walls_stay_stud_wall(self) -> None:
"""Regression: a Detailed §3.10 lodgement (no "Roof room type", per-
surface slopes lodged) keeps common walls as `stud_wall` — pinned
alongside `test_room_in_roof_reaches_building_part`."""
# Arrange — the Detailed fixture has no roof_room_type.
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["rooms_in_roof"] = True
data["roof_space"]["main_building"]["room_in_roof"] = self._RIR
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
rir = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[
0
].sap_room_in_roof
# Assert
assert rir is not None
assert rir.detailed_surfaces is not None
assert {s.kind for s in rir.detailed_surfaces if "wall" in s.kind} == {
"gable_wall_external",
"gable_wall",
"stud_wall",
}
def test_no_rooms_in_roof_stays_absent(self) -> None:
# Arrange — the fixture as lodged (rooms_in_roof=false).
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_building_parts[0].sap_room_in_roof is None
def test_insulation_not_threaded_when_surveyor_answered_not_known(
self,
) -> None:
"""On the survey form that asks per surface "Are you able to provide
details of the insulation …?", a "No" means the surveyor could NOT
confirm insulation for that surface (fixture 499617935574: loft hatch
screwed shut) — the roof-space thickness must not be threaded into it,
leaving the Table 18 col(4) age-band "unknown" default. Common walls
are never asked on this form, so they stay unthreaded too. The
legacy form (no question lodged anywhere; the `_RIR` fixture above)
keeps threading — pinned by `test_room_in_roof_reaches_building_part`.
"""
# Arrange — the verbatim 499617935574 answers: gables Yes, slopes and
# flat ceiling No, common walls never asked.
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["rooms_in_roof"] = True
rir = json.loads(json.dumps(self._RIR))
for gable in rir["gables"]:
gable["insulation_known"] = True
for slope in rir["slopes"]:
slope["insulation_known"] = False
rir["flat_ceilings"][0]["insulation_known"] = False
data["roof_space"]["main_building"]["room_in_roof"] = rir
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
mapped = EpcPropertyDataMapper.from_site_notes(survey).sap_building_parts[
0
].sap_room_in_roof
# Assert
assert mapped is not None
surfaces = mapped.detailed_surfaces
assert surfaces is not None
assert [(s.kind, s.insulation_thickness_mm) for s in surfaces] == [
("gable_wall_external", None),
("gable_wall", None),
("slope", None),
("slope", None),
("stud_wall", None),
("stud_wall", None),
("flat_ceiling", None),
]
class TestCylinderInsulationTypeCoding:
"""The surveyed cylinder "Insulation Type:" label must be int-coded for
the calculator's `cylinder_insulation_type` cascade ("Factory fitted" →
foam, code 1, mirroring Elmhurst "Foam"). Passed through raw, the
storage-loss resolver's `_int_or_none` yields None and a real lodged
cylinder is billed ZERO storage loss (fixture 507644414148 over-rates
by 1.4 SAP on this alone)."""
def test_factory_fitted_codes_to_foam(self) -> None:
# Arrange — the fixture as lodged ("Factory fitted", Normal cylinder).
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_heating is not None
assert result.sap_heating.cylinder_insulation_type == 1
def test_unknown_label_strict_raises(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["heating_and_hot_water"]["water_heating"]["insulation_type"] = (
"Sheep's wool wrap"
)
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act / Assert
with pytest.raises(UnmappedPasHubLabel):
EpcPropertyDataMapper.from_site_notes(survey)
class TestWindowOrientationCoding:
"""`from_site_notes` must int-code the PAS Hub window `orientation` label
to the SAP 10.2 octant code (1..8) the solar-gains cascade consumes.
`_orientation` in `solar_gains.py` returns None for a non-int and the
window is then skipped for solar gain entirely — so a raw string zeroes
EVERY window's solar contribution (100% of the Guinness cohort; the
single largest accuracy lever). PasHub lodges it space-separated
("South East") where the shared octant map keys hyphenated.
"""
@pytest.mark.parametrize(
"label, expected_code",
[
("North", 1),
("North East", 2),
("East", 3),
("South East", 4),
("South", 5),
("South West", 6),
("West", 7),
("North West", 8),
],
)
def test_label_maps_to_octant(
self, label: str, expected_code: int
) -> None:
data = load("pashub_rdsap_site_notes_example1.json")
data["windows"][0]["orientation"] = label
survey = from_dict(PasHubRdSapSiteNotes, data)
result = EpcPropertyDataMapper.from_site_notes(survey)
assert result.sap_windows[0].orientation == expected_code
def test_blank_orientation_stays_none(self) -> None:
# A window with no lodged orientation degrades to None (the cascade
# skips it) — NOT a strict-raise, NOT a default octant.
data = load("pashub_rdsap_site_notes_example1.json")
data["windows"][0]["orientation"] = ""
survey = from_dict(PasHubRdSapSiteNotes, data)
result = EpcPropertyDataMapper.from_site_notes(survey)
assert result.sap_windows[0].orientation is None
def test_unknown_orientation_strict_raises(self) -> None:
data = load("pashub_rdsap_site_notes_example1.json")
data["windows"][0]["orientation"] = "Sou'-sou'-west"
survey = from_dict(PasHubRdSapSiteNotes, data)
with pytest.raises(UnmappedPasHubLabel):
EpcPropertyDataMapper.from_site_notes(survey)
class TestWallDryLinedMapping:
"""A surveyed "Wall Dry-Lined? Yes" must reach
`sap_building_parts[*].wall_dry_lined` — the flag the wall-U cascade
reads for the RdSAP10 §5.8 / Table 14 adjustment (1/(1/U + 0.17)).
Dropped, a dry-lined wall is billed at the full uninsulated Table 6 U
(19% of the Guinness accuracy cohort lodges a "Yes"; already mapped on
the Elmhurst/API path, silently None on the pashub site-notes path)."""
def test_main_building_answer_reaches_building_part(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["building_construction"]["main_building"]["dry_lined"] = True
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_building_parts[0].wall_dry_lined is True
def test_extension_answer_reaches_building_part(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example2.json")
data["building_construction"]["extensions"][0]["dry_lined"] = False
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
extension_part = next(
bp
for bp in result.sap_building_parts
if bp.identifier != BuildingPartIdentifier.MAIN
)
assert extension_part.wall_dry_lined is False
def test_not_asked_stays_unknown(self) -> None:
# Arrange — the fixture as lodged (no "Wall Dry-Lined?" answer).
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert — unknown, NOT a "No".
assert result.sap_building_parts[0].wall_dry_lined is None
class TestSystemBuildWallIsNotBasement:
"""RdSAP wall-construction code 6 is overloaded: PasHub "System Build
(i.e Any Other)" maps to 6, but 6 doubles as the gov-API basement-wall
sentinel — with `wall_is_basement` left None, `main_wall_is_basement`
falls back to the code check and silently routes the dwelling through the
basement wall/floor cascade (issue #1590 bug 7). The mapper must pin the
flag False: a surveyed system-build wall is never a basement lodging.
"""
def test_system_build_wall_does_not_route_to_basement(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["building_construction"]["main_building"][
"walls_construction_type"
] = "System Build (i.e Any Other)"
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert — code 6, but explicitly NOT a basement.
part = result.sap_building_parts[0]
assert part.wall_construction == 6
assert part.main_wall_is_basement is False
class TestCylinderSizeCoding:
"""The surveyed §Water-Heating "Cylinder Size" label must resolve to the
SAP10 cylinder-size enum (`_cylinder_volume_l_from_code` int-or-nones a raw
string, silently skipping the Table 28 volume convention — issue #1590
bug 6). "No Access" routes per RdSAP 10 Table 28 p.55: solid-fuel boiler →
Medium/160L, off-peak dual electric immersion → Large/210L, otherwise →
Normal/110L (the cohort's case: cylinder fed from a gas main).
"""
@staticmethod
def _with_cylinder(label: str) -> Dict[str, Any]:
data = load("pashub_rdsap_site_notes_example1.json")
data["heating_and_hot_water"]["water_heating"]["cylinder_size"] = label
return data
@pytest.mark.parametrize(
"label, expected_code",
[
("Normal (90-130 litres)", 2),
("Large (>170 litres)", 4),
("No Access", 2), # fed from the (gas) main → Table 28 "otherwise"
],
)
def test_label_maps_to_sap_code(self, label: str, expected_code: int) -> None:
# Arrange
survey = from_dict(PasHubRdSapSiteNotes, self._with_cylinder(label))
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_heating.cylinder_size == expected_code
def test_no_cylinder_stays_absent(self) -> None:
# Arrange — a combi dwelling: no cylinder lodged.
survey = from_dict(
PasHubRdSapSiteNotes, self._with_cylinder("No Cylinder")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_heating.cylinder_size is None
def test_unknown_label_strict_raises(self) -> None:
# Arrange
survey = from_dict(
PasHubRdSapSiteNotes, self._with_cylinder("Some novel size")
)
# Act / Assert
with pytest.raises(UnmappedPasHubLabel, match="cylinder size"):
EpcPropertyDataMapper.from_site_notes(survey)
class TestMechanicalVentilationKindCoding:
"""The surveyed "Ventilation type" label must resolve to the
`MechanicalVentilationKind` enum name the §2 cascade dispatches on —
left None, `ventilation_from_cert` silently defaults every mechanical
system to NATURAL (24d), dropping its ACH heat loss (issue #1590 bug 4;
76/205 cohort fixtures lodge a mechanical system). Targets mirror the
gov-API table `_API_MECHANICAL_VENTILATION_TO_KIND` (dMEV/MEV/PIV-outside
→ EXTRACT_OR_PIV_OUTSIDE; PIV-from-loft is "as natural" → None).
"""
@pytest.mark.parametrize(
"label, expected_kind",
[
("Natural", None),
("Mechanical Extract - Decentralised", "EXTRACT_OR_PIV_OUTSIDE"),
("Mechanical Extract - Centralised", "EXTRACT_OR_PIV_OUTSIDE"),
("Positive Input - From Outside", "EXTRACT_OR_PIV_OUTSIDE"),
("Positive Input - From Loft (As Natural)", None),
],
)
def test_label_maps_to_mechanical_kind(
self, label: str, expected_kind: Optional[str]
) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["ventilation"]["ventilation_type"] = label
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_ventilation is not None
assert result.sap_ventilation.mechanical_ventilation_kind == expected_kind
def test_unknown_label_strict_raises(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["ventilation"]["ventilation_type"] = "Some novel system"
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act / Assert
with pytest.raises(UnmappedPasHubLabel, match="ventilation type"):
EpcPropertyDataMapper.from_site_notes(survey)
class TestPvConnectionCoding:
"""The surveyed "PV Connection" label must resolve to the gov-API int enum
the calculator's credit gate keys off (`_pv_connected_to_dwelling_meter`:
only int 2 earns the Appendix M credit; int 1 is "present but NOT connected"
— zero credit). The raw string fell through the gate's non-int branch to
True, crediting PV a dwelling doesn't get to count (issue #1590 bug 3;
fixture 499516101839 over-rates by ~6.9 SAP).
"""
@pytest.mark.parametrize(
"label, expected_code",
[
("Not connected to electricity meter", 1),
("Connected to dwellings electricity meter", 2),
],
)
def test_label_maps_to_gov_api_code(
self, label: str, expected_code: int
) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["renewables"]["pv_connection"] = label
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_energy_source.pv_connection == expected_code
def test_absent_connection_stays_absent(self) -> None:
# Arrange — no PV Connection lodged (fixture default).
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_energy_source.pv_connection is None
def test_unknown_label_strict_raises(self) -> None:
# Arrange
data = load("pashub_rdsap_site_notes_example1.json")
data["renewables"]["pv_connection"] = "Some novel connection"
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act / Assert
with pytest.raises(UnmappedPasHubLabel, match="PV connection"):
EpcPropertyDataMapper.from_site_notes(survey)
class TestRoofInsulationNoneCoding:
"""A surveyed roof-space "Insulation At: None" is the assessor explicitly
recording ZERO loft insulation — `from_site_notes` must lodge an explicit
`roof_insulation_thickness=0`, not leave it `None` ("unknown"). With `None`
the calculator's `u_roof` falls through to the age-band "assumed insulated"
default (e.g. 0.16 W/m²K at band J/K) instead of the Table 16 uninsulated
row (2.30) — a ~14x understatement of roof heat loss (issue #1590 bug 2;
fixtures 507665533138 / 507670893756 over-rate by ~7 SAP each).
"""
def test_insulation_at_none_lodges_explicit_zero_thickness(self) -> None:
# Arrange — the surveyor recorded "Insulation At: None" (no thickness
# line follows; there is no insulation to measure).
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["insulation_at"] = "None"
data["roof_space"]["main_building"]["insulation_thickness_mm"] = None
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert — explicit zero, so u_roof takes the uninsulated branch.
assert result.sap_building_parts[0].roof_insulation_thickness == 0
def test_surveyed_thickness_still_passes_through(self) -> None:
# Arrange — the fixture's as-lodged "Joists" + 100mm.
survey = from_dict(
PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json")
)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_building_parts[0].roof_insulation_thickness == 100
def test_blank_insulation_at_stays_unknown(self) -> None:
# Arrange — a blank lodging is "not surveyed", NOT "no insulation";
# it must stay None so the age-band default still applies.
data = load("pashub_rdsap_site_notes_example1.json")
data["roof_space"]["main_building"]["insulation_at"] = ""
data["roof_space"]["main_building"]["insulation_thickness_mm"] = None
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
assert result.sap_building_parts[0].roof_insulation_thickness is None
def test_extension_insulation_at_none_lodges_explicit_zero(self) -> None:
# Arrange — same behaviour on the extension building part.
data = load("pashub_rdsap_site_notes_example2.json")
data["roof_space"]["extensions"][0]["insulation_at"] = "None"
data["roof_space"]["extensions"][0]["insulation_thickness_mm"] = None
survey = from_dict(PasHubRdSapSiteNotes, data)
# Act
result = EpcPropertyDataMapper.from_site_notes(survey)
# Assert
extension_part = next(
bp
for bp in result.sap_building_parts
if bp.identifier != BuildingPartIdentifier.MAIN
)
assert extension_part.roof_insulation_thickness == 0
class TestCountryCode:
"""`from_site_notes` must set `country_code` so the cascade's England /
Wales / Scotland / NI U-value tables (`u_floor`, `u_basement_floor`,
@ -169,6 +849,62 @@ class TestWallConstructionCoding:
EpcPropertyDataMapper.from_site_notes(survey)
class TestSecondaryHeatingTypeCoding:
"""`from_site_notes` must int-code the PAS Hub secondary-heating
`secondary_system` label to the SAP 10.2 Table 4a room-heater code, not
pass the raw string through. `_secondary_efficiency` does
`_int_or_none(secondary_heating_type)` — a string yields None and
silently falls back to code 693 (portable electric heater, eff 1.00),
so a gas open fire (code 605, eff 0.40) is billed as 100%-efficient
electric. 70/205 cohort properties lodge a secondary system; the 3
gas/solid-fuel room heaters (e.g. deal 499607224524, +4.0 SAP) were
over-credited by the wrong fallback.
"""
@pytest.mark.parametrize(
"label, expected_code",
[
# Electric direct-acting (PCDB 691, eff 1.00) — the modal cohort
# heater; 691 == the old 693 fallback efficiency, so unchanged.
("Panel, convector or radiant heaters", 691),
# Gas room heaters (PCDB Table 4a column A).
(
"Flush fitting live fuel effect open front fire, "
"sealed to fireplace",
605,
),
(
"1980 or later - Open front fire with open flue, "
"sealed to fireplace",
603,
),
# Solid fuel (PCDB 631, eff 0.32 col B).
("Open fire in grate", 631),
("No Secondary Heating", None), # no secondary heater lodged
("", None), # no lodging
],
)
def test_label_maps_to_sap_code(
self, label: str, expected_code: Optional[int]
) -> None:
data = load("pashub_rdsap_site_notes_example1.json")
data["heating_and_hot_water"]["secondary_heating"][
"secondary_system"
] = label
survey = from_dict(PasHubRdSapSiteNotes, data)
result = EpcPropertyDataMapper.from_site_notes(survey)
assert result.sap_heating.secondary_heating_type == expected_code
def test_unknown_label_strict_raises(self) -> None:
data = load("pashub_rdsap_site_notes_example1.json")
data["heating_and_hot_water"]["secondary_heating"][
"secondary_system"
] = "Wood-burning stove with heat battery"
survey = from_dict(PasHubRdSapSiteNotes, data)
with pytest.raises(UnmappedPasHubLabel):
EpcPropertyDataMapper.from_site_notes(survey)
class TestSecondaryFuelTypeCoding:
"""`from_site_notes` must int-code the PAS Hub secondary-heating
`secondary_fuel` label to the SAP 10.2 Table 32 `secondary_fuel_type`
@ -181,6 +917,9 @@ class TestSecondaryFuelTypeCoding:
"label, expected_code",
[
("Electricity", 30), # standard electricity, Table 32 code 30
("Mains gas", 26), # mains-gas secondary heater, code 26
("Mains Gas", 26), # capitalised site-note variant, same code
("House Coal", 11), # solid-fuel secondary, Table 32 house coal
("No Secondary Heating", None), # no secondary heater lodged
("", None), # no lodging
],
@ -570,7 +1309,9 @@ class TestFromSiteNotesExample1:
assert result.sap_windows[0].draught_proofed is True
def test_window_orientation(self, result: EpcPropertyData) -> None:
assert result.sap_windows[0].orientation == "South East"
# "South East" → SAP10 octant code 4 (the solar-gains cascade reads
# orientation as an int; a raw string silently zeroes the gain).
assert result.sap_windows[0].orientation == 4
def test_window_glazing_type(self, result: EpcPropertyData) -> None:
# windows[].glazing_type: "Double glazing, Unknown install date"
@ -763,8 +1504,8 @@ class TestFromSiteNotesExample1:
)
],
has_fixed_air_conditioning=False,
cylinder_size="Normal (90-130 litres)",
cylinder_insulation_type="Factory fitted",
cylinder_size=2, # "Normal (90-130 litres)" → Table 28 code 2
cylinder_insulation_type=1, # "Factory fitted" → foam, SAP10 code 1
cylinder_insulation_thickness_mm=12,
water_heating_code=901, # cylinder + "From main heating 1" → WHC 901
shower_outlets=ShowerOutlets(
@ -778,7 +1519,7 @@ class TestFromSiteNotesExample1:
SapWindow(
frame_material="Wooden or PVC",
glazing_gap="16 mm or more",
orientation="South East",
orientation=4, # "South East" → SAP10 octant 4
window_type="Window",
glazing_type=3, # "Double glazing, Unknown install date" → cascade code 3
window_width=1.0,
@ -791,7 +1532,7 @@ class TestFromSiteNotesExample1:
SapWindow(
frame_material="Wooden or PVC",
glazing_gap="16 mm or more",
orientation="South East",
orientation=4, # "South East" → SAP10 octant 4
window_type="Window",
glazing_type=3, # "Double glazing, Unknown install date" → cascade code 3
window_width=0.96,
@ -804,7 +1545,7 @@ class TestFromSiteNotesExample1:
SapWindow(
frame_material="Wooden or PVC",
glazing_gap="16 mm or more",
orientation="North West",
orientation=8, # "North West" → SAP10 octant 8
window_type="Window",
glazing_type=3, # "Double glazing, Unknown install date" → cascade code 3
window_width=0.96,
@ -817,7 +1558,7 @@ class TestFromSiteNotesExample1:
SapWindow(
frame_material="Wooden or PVC",
glazing_gap="16 mm or more",
orientation="North West",
orientation=8, # "North West" → SAP10 octant 8
window_type="Window",
glazing_type=3, # "Double glazing, Unknown install date" → cascade code 3
window_width=0.97,
@ -844,6 +1585,7 @@ class TestFromSiteNotesExample1:
SapBuildingPart(
identifier=BuildingPartIdentifier.MAIN,
construction_age_band="I",
wall_is_basement=False, # PasHub never lodges a basement wall
wall_construction=4, # "Cavity" → SAP10 code 4 (WALL_CAVITY)
wall_insulation_type=4, # "As built" → SAP10 code 4 (as built/assumed)
wall_thickness_measured=True,
@ -1266,18 +2008,19 @@ class TestPasHubMainHeatingControlCoding:
# inventory `_CONTROL_TYPE_BY_CODE`; a code that isn't there would
# strict-raise `UnmappedSapCode` in `_control_type`. This guards the
# mapper against drifting out of sync with the calculator's coverage.
from datatypes.epc.domain.mapper import ( # pyright: ignore[reportPrivateUsage]
_PASHUB_MAIN_HEATING_CONTROL_TO_SAP10,
from datatypes.epc.domain.mapper import (
_PASHUB_HEAT_NETWORK_CONTROL_TO_SAP10, # pyright: ignore[reportPrivateUsage]
_PASHUB_MAIN_HEATING_CONTROL_TO_SAP10, # pyright: ignore[reportPrivateUsage]
)
from domain.sap10_calculator.rdsap.cert_to_inputs import ( # pyright: ignore[reportPrivateUsage]
_CONTROL_TYPE_BY_CODE,
from domain.sap10_calculator.rdsap.cert_to_inputs import (
_CONTROL_TYPE_BY_CODE, # pyright: ignore[reportPrivateUsage]
)
unknown = {
code
for code in _PASHUB_MAIN_HEATING_CONTROL_TO_SAP10.values()
if code not in _CONTROL_TYPE_BY_CODE
mapped_codes = {
*_PASHUB_MAIN_HEATING_CONTROL_TO_SAP10.values(),
*_PASHUB_HEAT_NETWORK_CONTROL_TO_SAP10.values(),
}
unknown = {c for c in mapped_codes if c not in _CONTROL_TYPE_BY_CODE}
assert not unknown
@ -1358,6 +2101,52 @@ class TestPasHubMainHeatingControlCoding:
with pytest.raises(UnmappedPasHubLabel, match="Heat pump"):
EpcPropertyDataMapper.from_site_notes(survey)
def test_heat_network_control_maps_to_group3_code(self) -> None:
# Arrange — a Table 4e Group 3 heat network (`system_type =
# "Community heating system"`). Its control label carries a Group 3
# (23xx) code, not the boiler 21xx; the accredited Elmhurst certs for
# this same cohort lodge 2306 for it (HANDOVER_POST_S0380_176).
from domain.sap10_calculator.rdsap.cert_to_inputs import (
_control_type, # pyright: ignore[reportPrivateUsage]
)
raw = load("pashub_rdsap_site_notes_example1.json")
mh = raw["heating_and_hot_water"]["main_heating"]
mh["system_type"] = "Community heating system"
# a complete community lodgement (heat source + fuel) so the control
# resolution is what's exercised, not the community strict-raise
mh["community_heat_source"] = "Community heating - boilers only"
mh["fuel"] = "Mains Gas"
mh["product_id"] = 0
mh["controls"] = (
"Charging system linked to use of community heating, room thermostat only"
)
survey = from_dict(PasHubRdSapSiteNotes, raw)
# Act
main = EpcPropertyDataMapper.from_site_notes(
survey
).sap_heating.main_heating_details[0]
# Assert — the Group 3 code, and its spec control type (3).
assert main.main_heating_control == 2306
assert _control_type(main) == 3
def test_heat_network_unmapped_control_label_strict_raises(self) -> None:
# A community-system control label the Group 3 lookup does not cover
# strict-raises (naming label + system), never silently mis-rating.
raw = load("pashub_rdsap_site_notes_example1.json")
mh = raw["heating_and_hot_water"]["main_heating"]
mh["system_type"] = "Community heating system"
mh["community_heat_source"] = "Community heating - boilers only"
mh["fuel"] = "Mains Gas"
mh["product_id"] = 0
mh["controls"] = "Some novel control"
survey = from_dict(PasHubRdSapSiteNotes, raw)
with pytest.raises(UnmappedPasHubLabel, match="Community heating system"):
EpcPropertyDataMapper.from_site_notes(survey)
class TestPasHubUnmappedHeatEmitter:
"""A PasHub survey whose surveyed main-heating emitter label the mapper does

60
datatypes/epc/surveys/pashub_rdsap_site_notes.py
View file

@ -47,6 +47,9 @@ class MainBuildingConstruction:
wall_thickness_mm: Optional[int]
party_wall_construction_type: str
filled_cavity_indicators: Optional[str] = None
# "Wall Dry-Lined?" Yes/No; None when the survey doesn't ask (e.g.
# non-masonry system build).
dry_lined: Optional[bool] = None
@dataclass
@ -62,6 +65,8 @@ class ExtensionConstruction:
wall_thickness_mm: Optional[int]
party_wall_construction_type: str
filled_cavity_indicators: Optional[str] = None
# "Wall Dry-Lined?" Yes/No; None when the survey doesn't ask.
dry_lined: Optional[bool] = None
@dataclass
@ -105,6 +110,40 @@ class BuildingMeasurements:
extensions: Optional[List[ExtensionMeasurements]] = None
@dataclass
class RoomInRoofSurfaceDetail:
"""One surveyed room-in-roof surface (gable / slope / common wall / flat
ceiling): length × height, an optional measured U, and — for gables —
the Exposed/Party type. Raw survey values; SAP coding happens at the
mapper boundary (ADR-0015)."""
length_m: Optional[float] = None
height_m: Optional[float] = None
u_value: Optional[float] = None
gable_type: Optional[str] = None
# The surveyor's per-surface "{surface}: Are you able to provide details
# of the insulation …?" Yes/No; None when the form didn't ask (legacy
# forms, and common walls — never asked on any form).
insulation_known: Optional[bool] = None
@dataclass
class RoomInRoofDetail:
"""The surveyed §Roof-Space room-in-roof block (RdSAP §3.10 detailed
measurement): age range, floor area, and the per-surface detail."""
age_range: Optional[str] = None
floor_area_m2: Optional[float] = None
gables: Optional[List[RoomInRoofSurfaceDetail]] = None
slopes: Optional[List[RoomInRoofSurfaceDetail]] = None
common_walls: Optional[List[RoomInRoofSurfaceDetail]] = None
flat_ceilings: Optional[List[RoomInRoofSurfaceDetail]] = None
# The "Roof room type:" label, lodged only on the Simplified approach
# ("Are details of the room in roof known? No" → RdSAP §3.9.2 Type 1/2/3).
# None on a Detailed §3.10 lodgement (per-surface slopes/flat ceilings).
roof_room_type: Optional[str] = None
@dataclass
class RoofSpaceDetail:
construction_type: str
@ -115,6 +154,7 @@ class RoofSpaceDetail:
# Numeric thickness (mm) when known; string (e.g. "As built") when not measured
insulation_thickness_mm: Optional[int] = None
insulation_thickness: Optional[str] = None
room_in_roof: Optional[RoomInRoofDetail] = None
@dataclass
@ -127,6 +167,7 @@ class ExtensionRoofSpace:
rooms_in_roof: bool
insulation_thickness_mm: Optional[int] = None
insulation_thickness: Optional[str] = None
room_in_roof: Optional[RoomInRoofDetail] = None
@dataclass
@ -174,6 +215,10 @@ class MainHeating:
weather_compensator: bool
emitter: str
emitter_temperature: str
# §Heating "Heating System (Other)" — the community heat-source label
# (e.g. "Community heating - boilers only") lodged only by heat-network
# dwellings; blank elsewhere.
community_heat_source: str = ""
@dataclass
@ -222,6 +267,19 @@ class Conservatories:
has_conservatory: bool
@dataclass
class PvArrayDetail:
"""One surveyed PV array from the Renewables section's per-system block
("PV {n}: kWp value / Photovoltaic Pitch / Orientation / Overshading").
Raw survey labels for orientation/overshading — coding to SAP ints happens
at the mapper boundary (ADR-0015)."""
kwp: Optional[float] = None
pitch_degrees: Optional[int] = None
orientation: Optional[str] = None
overshading: Optional[str] = None
@dataclass
class Renewables:
wind_turbines: bool
@ -231,6 +289,8 @@ class Renewables:
hydro: bool
pv_connection: Optional[str] = None
percent_roof_covered_pv: Optional[int] = None
pv_diverter_present: Optional[bool] = None
pv_arrays: Optional[List[PvArrayDetail]] = None
@dataclass

86
docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md Normal file
View file

@ -0,0 +1,86 @@
# PasHub portfolio 838 — problem properties & validation (2026-07-14)
Cohort: **The Guinness Partnership GMCA** (205 PasHub site-notes properties),
portfolio **838**, scenario **1297**. Modelled locally through the real
`applications/modelling_e2e` lambda handler using the **stored** PasHub site
notes (`refetch_epc=False`) — see `scripts/run_pashub_modelling.py`.
## Headline: our SAP calculator diverges from pashub's rating by ~MAE 2.7 — real work remains
**Do not use `property_baseline_performance.effective_sap_score` to validate the
calculator.** For a SAP-10.2 lodged EPC whose physical state is unchanged, the
rebaseliner passes **Lodged Performance through as Effective**
(`domain/property_baseline/rebaseliner.py`), and Lodged Performance's `sap_score`
is `energy_rating_current` — the fetched pashub rating
(`domain/property_baseline/performance.py:63`). So `effective_sap_score` **is**
the pashub rating; comparing it to `energy_rating_current` compares the rating to
itself (circular — an earlier "99.5% match" was this mistake).
The real accuracy of our **calculator** is the `test_pashub_sap_accuracy`
harness: **~12.2% within-0.5, MAE ~2.7** vs `pre_sap` (which ≈ the pashub
rating, MAE 0.15 — same ground truth). Worked example: property **754881** — our
extraction is **byte-identical** to the stored ingestion (fuel 26, PCDB index
18119, wall 4/2, roof 200mm, party-wall 4, floor solid), yet our calculator
scores **71.9** where pashub lodged **85** — a genuine ~13-point gap to
root-cause. These are the extraction/calculator bugs to hunt.
**Image stripping does NOT lose data:** the fixture (stripped) and DB (original
PDF) parses of 754881 are identical field-for-field — so the harness fixtures are
faithful and the ~2.7 MAE is a true calculator gap, not a fixture artifact.
`parse_site_notes_pdf` never extracts the SAP rating from the PDF at all
(`energy_rating_current` is populated by Dan's separate pashub-API fetch,
`pashub_service.py:45` ← `preSapRating`), which is why the rating is `None` on a
raw parse but present in the DB — unrelated to stripping.
## Properties that did not run (need PasHub re-extraction — data, not code)
These 3 have **no fresh int-coded stored site note** (only pre-fix rows with a
string fuel), so modelling strict-raised and skipped them. Re-trigger PasHub
extraction so a current-mapper site note is stored, then they model like the
other 201.
| property_id | UPRN | address | stored site-note fuel | fix |
|---|---|---|---|---|
| 754772 | 77168847 | 9 Philips Park Court, Willdale Close, M11 4DH | `"Mains gas"` (string) | re-extract |
| 754780 | 77180607 | 12 Seymour Road South, Clayton, M11 4PG | `"Mains gas"` (string) | re-extract |
| 754844 | 77155031 | 16 Bingley Close, Beswick, M11 3RF | `""` (blank) | re-extract; if the survey genuinely lodges no main fuel, that is a separate blank/residual-fuel mapper gap |
## Properties that ran but not from their own PasHub survey
| property_id | UPRN | address | note |
|---|---|---|---|
| 754816 | 77180053 | 130 Stanton Street, Clayton, M11 4PX | **no PasHub site note exists at all** — modelled via the prediction path. Needs extraction so it models from its own survey. |
| 754778 | 77181049 | (M11) | winner lodged EPC is **not** a PasHub site note (gov EPC / other); modelled SAP 60 vs stored rating 57 (**d=+3**, the cohort's only >0.5 divergence). Check why its PasHub site note isn't the winner. |
## Accuracy outliers surfaced by closing the extractor gaps
Closing the last 4 `test_pashub_sap_accuracy` xfails (see the PR) makes them
**compute**; two then show a large gap **vs `pre_sap`** (not vs the stored
rating — verify `pre_sap` first, do not tune to it):
| fixture (deal) | ours | pre_sap | note |
|---|---|---|---|
| 499584755922 | 34.0 | 67 | main fuel extracted as **Bulk LPG (27)** + house-coal secondary → low SAP. Either a bad `pre_sap` or a main-fuel extraction issue; confirm the survey's main fuel. |
| 507644414148 | 70.2 | 52 | **community heating**. The PasHub path maps only the Table 4e Group 3 control code (2306); it does **not** yet set the full heat-network fuel/flags (main_fuel is 26, not a Table 12 community code). Deeper community-heating mapping gap. |
## Root-caused extractor bugs (issue #1590)
Five worst-divergence properties were deep-dived (extracted inputs diffed
against the PDF text; gaps attributed via patch-and-rerun). **7 distinct
extractor bugs** — PV arrays never extracted (−13.5), roof "Insulation At:
None" treated as unknown not zero (−7.6/−7.1, systematic), `pv_connection`
string passthrough (−6.9), ventilation kind never mapped (−4.7), room-in-roof
never built (−2.1), cylinder "No Access" passthrough (−0.6), system-build/
basement code-6 collision (latent) — are itemised with fixes and fixture deals
in **https://github.com/Hestia-Homes/Model/issues/1590**. Also there: the
ground-truth caveat that `pre_sap` is pashub's *preliminary* figure — the
accredited lodgement can differ downstream of the site notes (754917: 53 → 43).
## How to reproduce
```
python scripts/run_pashub_modelling.py # dry-run, all 838 batches
RUN_DRY=0 python scripts/run_pashub_modelling.py # real writes
RUN_PIDS="754772" RUN_DRY=0 python scripts/run_pashub_modelling.py # one property
```
Validation sweep (ours vs stored rating) is in the PR description / this handover.

99
docs/HANDOVER_PASHUB_EXTRACTOR_ACCURACY.md Normal file
View file

@ -0,0 +1,99 @@
# PasHub extractor accuracy — handover (2026-07-14 late)
## Goal
Guinness GMCA 205-property cohort within 0.5 SAP of pashub's `pre_sap`
(PRD #1555). Khalim: "We can get to 100% within 0.5 for this cohort." The
bugs are **extraction** (survey detail not reaching the calculator), not the
calculator — proven repeatedly this session.
## Where we are (branch `feat/pashub-extractor-fixes-and-runner`, pushed)
Harness `backend/documents_parser/tests/test_pashub_sap_accuracy.py`:
**205/205 compute, MAE 1.551, within-0.5 18.5% (38/205), bias ≈ −1.0**
(from 12.4%/2.56 at session start). Ratchets `_MIN_WITHIN_HALF=0.18`,
`_MAX_SAP_MAE=1.56` — tighten after every fix, comment each move.
### Fixed this session (all TDD 🟥/🟩 pairs; see git log on the branch)
Issue **#1590** (all 7 subtasks done + 2 round-2 fixes; trajectory table in
its last comment): secondary fuel labels; Table 4e Group 3 control 2306;
roof "Insulation At: None"→explicit 0; **PV arrays extracted+mapped**
(`_pashub_pv_arrays`); `pv_connection`→int (1/2); **mechanical ventilation**
`_pashub_mechanical_ventilation_kind` (biggest win: 76 fixtures, MAE −0.8);
cylinder size Table 28; system-build/basement code-6 collision;
**room-in-roof** (survey `RoomInRoofDetail` → extractor `_parse_room_in_roof`
→ mapper `_pashub_room_in_roof`; commons are `stud_wall`, roof-space
insulation threads into slope/stud/flat surfaces); **community heating**
(extractor captures "Heating System (Other):" → `_pashub_community_heating`
→ `sap_main_heating_code=301` + Table 12 fuel 51 via
`_resolve_community_heating_fuel_code`).
Issue **#1589** (separate, unfixed): modelling `refetch_epc=True` violates
ADR-0001 recency — gov API overrides newer stored site notes.
## Ground truth — now verified
`pre_sap` in the manifest was hand-verified by Khalim in pashub for the 4
worst deviations (2026-07-14): manifest corrected for 499584755922 (F33, was
stale D67 — we were right) and 499617935574 (accredited E43, was preliminary
53). **Confirmed correct** (so our deviation = extraction bug to hunt):
| deal | address | pashub | ours | d |
|---|---|---|---|---|
| 499530733767 | 58 Hackle Street, M11 4WU | D58 | 65.8 | **+7.8** |
| 499617935574 | Brightholme, 14 North Road, M11 4WE | 43 | 50.3 | **+7.3** |
| 507644414148 | 16 Bingley Close, M11 3RF | E52 | 57.7 | **+5.7** |
Rule: don't tune to `pre_sap` blindly, but after this verification treat it
as trustworthy unless a survey PDF plainly contradicts it (then ask Khalim to
check pashub — give him the ADDRESS, not the deal id).
## The proven working loop (repeat until 100%)
1. Rank divergences: parse every manifest fixture (`parse_site_notes_pdf` →
`Sap10Calculator().calculate`) vs `pre_sap_score`; list worst signed.
2. Fan out 3-5 Sonnet sub-agents, one per worst case, with THE RECIPE:
dump extracted `EpcPropertyData` fields; dump PDF text (`fitz`); diff
field-by-field against the survey; attribute via `dataclasses.replace`
patch-and-rerun (record SAP delta per candidate fix); report the
mis-extracted field + concrete fix + deltas. Investigation only, no edits.
3. TDD each confirmed fix (`/tdd` skill): RED commit 🟥 → GREEN commit 🟩,
test in `datatypes/epc/domain/tests/test_from_site_notes.py` (mapper) and/or
`backend/documents_parser/tests/test_extractor.py` (extractor; splice
verbatim PDF lines into `load_text_fixture()`); update the golden
`test_full_mapping` when field coding changes.
4. Re-run harness; ratchet; commit; push.
## Conventions (non-negotiable)
- `_pashub_*` helpers strict-raise `UnmappedPasHubLabel` on unknown non-empty
labels; blank passes through/None. Mirror `_ELMHURST_*`/`_api_*` siblings.
- pyright: zero NEW errors (baselines: mapper.py 39, extractor.py 34,
test_from_site_notes.py ≤13). Beware `Union[RoofSpaceDetail,
ExtensionRoofSpace]` attribute access — add fields to BOTH.
- Survey dataclasses hold RAW labels; coding happens at the mapper boundary
(ADR-0015). Ratchets never loosen at fixed coverage.
## Known leads for the remaining gap
- **Solid ground floors**: worst-biased category (n=107, mean −1.53; 5/7 of
the −4..−5 Natural-ventilation cluster: 499529160912, 499538003156,
499627236560, 499586692333, 507533541571, 499570197710, 499592219845).
Dig the ISO 13370 solid-floor U/perimeter cascade vs what the surveys lodge.
- 16 Bingley (+5.7): community-heating residual — water heating is
"Hot water only community scheme - boilers" (WHC 950 path?), community
standing charge/tariff detail, or DLF "Unknown" distribution type.
- 58 Hackle (+7.8): undug — full recipe pass needed.
- Brightholme (+7.3): RIR dwelling; the accredited 43 reflects a QA
correction ("no access to loft — hatch screwed shut") — possibly RIR
insulation should NOT thread from the roof-space block when access was
impossible, or slopes should default worse. Check `_pashub_room_in_roof`.
- Mechanical ventilation and PCDB efficiency are VERIFIED CORRECT — don't re-dig.
## Also outstanding
- **Open the review PR** for the branch (metrics table = #1590's last
comment; ~26 commits, all green: 206 harness + ~294 unit tests).
- Portfolio 838 modelling: re-extract 4 properties in pashub (754772/9
Philips Park Court, 754780/12 Seymour Rd S, 754844/16 Bingley — the
community fix now makes it extractable, 754816/130 Stanton St has no site
note at all); then `RUN_DRY=0 python scripts/run_pashub_modelling.py`
(env-parametrised: RUN_PORTFOLIO/RUN_SCENARIO/RUN_PIDS/RUN_DRY).
- `effective_sap_score` == the pashub rating for unchanged lodged dwellings
(rebaseliner pass-through) — NEVER validate the calculator against it.
- hubspot_deal_data still holds the stale pre_sap for the 2 corrected deals —
re-running `scripts/build_pashub_accuracy_fixtures.py` will REVERT the
manifest corrections unless the DB rows are fixed first.

122
scripts/run_pashub_modelling.py Normal file
View file

@ -0,0 +1,122 @@
"""Run the modelling_e2e handler locally over a portfolio's properties, using
the STORED EPCs (``refetch_epc=False``) — the correct source for PasHub
site-notes cohorts, where the stored site note is the newest assessment (the
gov-API refetch would bypass it; see issue #1589 / ADR-0001).
Invokes the real Lambda handler in-process (no Docker) with one SQS-shaped event
per batch of property ids. Reads ``backend/.env`` and maps ``DB_*`` -> the
``POSTGRES_*`` vars the handler's ``PostgresConfig.from_env`` expects.
Config via environment variables (all optional):
RUN_PORTFOLIO portfolio id (default 838)
RUN_SCENARIO scenario id (default 1297)
RUN_PIDS comma-separated property ids; unset -> every property in the
portfolio, ordered by id
RUN_DRY "1" dry-run / "0" write to the DB (default "1")
RUN_BATCH properties per handler invocation (default 50)
RUN_REFETCH_EPC / RUN_REPREDICT_EPC / RUN_REFETCH_SOLAR
"1"/"0" to override the refetch flags (default "0" — use
stored EPCs/solar; flip to re-fetch from the gov API/Google)
Examples:
python scripts/run_pashub_modelling.py # dry-run, whole portfolio
RUN_DRY=0 python scripts/run_pashub_modelling.py # real writes
RUN_SCENARIO=1301 RUN_DRY=0 python scripts/run_pashub_modelling.py
RUN_PIDS=754772,754780 RUN_DRY=0 python scripts/run_pashub_modelling.py
"""
from __future__ import annotations
import json
import os
from sqlalchemy import text
from scripts.e2e_common import build_engine, load_env
def _configure_env() -> None:
"""Load ``backend/.env`` and mirror the FastAPI-layer ``DB_*`` creds onto the
``POSTGRES_*`` names the modelling handler reads."""
load_env()
for pg, db in (
("POSTGRES_HOST", "DB_HOST"),
("POSTGRES_PORT", "DB_PORT"),
("POSTGRES_USERNAME", "DB_USERNAME"),
("POSTGRES_PASSWORD", "DB_PASSWORD"),
("POSTGRES_DATABASE", "DB_NAME"),
):
if db in os.environ:
os.environ.setdefault(pg, os.environ[db])
def _flag(name: str, default: bool) -> bool:
raw = os.environ.get(name)
return default if raw is None else raw == "1"
def _portfolio_property_ids(portfolio_id: int) -> list[int]:
engine = build_engine()
with engine.begin() as conn:
conn.execute(text("SET statement_timeout = 120000"))
rows = conn.execute(
text("SELECT id FROM property WHERE portfolio_id = :p ORDER BY id"),
{"p": portfolio_id},
)
return [int(r[0]) for r in rows]
def main() -> None:
_configure_env()
# Imported after env is configured so the handler's module-level engine binds
# to the mirrored POSTGRES_* vars.
from applications.modelling_e2e.handler import handler
portfolio_id = int(os.environ.get("RUN_PORTFOLIO", "838"))
scenario_id = int(os.environ.get("RUN_SCENARIO", "1297"))
dry_run = _flag("RUN_DRY", True)
batch_size = int(os.environ.get("RUN_BATCH", "50"))
refetch_epc = _flag("RUN_REFETCH_EPC", False)
repredict_epc = _flag("RUN_REPREDICT_EPC", False)
refetch_solar = _flag("RUN_REFETCH_SOLAR", False)
pids_env = os.environ.get("RUN_PIDS")
property_ids: list[int] = (
[int(x) for x in pids_env.split(",") if x.strip()]
if pids_env
else _portfolio_property_ids(portfolio_id)
)
batches: list[list[int]] = [
property_ids[i : i + batch_size]
for i in range(0, len(property_ids), batch_size)
]
print(
f"portfolio={portfolio_id} scenario={scenario_id} dry_run={dry_run} "
f"refetch_epc={refetch_epc} repredict_epc={repredict_epc} "
f"refetch_solar={refetch_solar} properties={len(property_ids)} "
f"batches={len(batches)}"
)
for i, chunk in enumerate(batches):
body = {
"property_ids": chunk,
"portfolio_id": portfolio_id,
"scenario_id": scenario_id,
"refetch_epc": refetch_epc,
"repredict_epc": repredict_epc,
"refetch_solar": refetch_solar,
"dry_run": dry_run,
}
event = {"Records": [{"body": json.dumps(body)}]}
print(
f"\n=== batch {i + 1}/{len(batches)}: {len(chunk)} props "
f"({chunk[0]}..{chunk[-1]}) ==="
)
result: object = handler(event, None)
print(" ->", json.dumps(result, default=str)[:800])
if __name__ == "__main__":
main()