diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 37eeb353e..a487f10de 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -1,5 +1,6 @@ import copy import logging +import math import re from dataclasses import replace from datetime import date @@ -3201,13 +3202,16 @@ class EpcPropertyDataMapper: ) return None - return _clear_basement_flag_when_system_built( - _with_renewable_heat_incentive( - _with_recorded_performance( - _drop_building_parts_without_age_band(mapped), data - ), - data, - ) + return _with_internal_dimensions( + _clear_basement_flag_when_system_built( + _with_renewable_heat_incentive( + _with_recorded_performance( + _drop_building_parts_without_age_band(mapped), data + ), + data, + ) + ), + data, ) @@ -3424,6 +3428,181 @@ def _sap_back_solved_habitable_rooms(schema: SapSchema17_1) -> int: ) +# --------------------------------------------------------------------------- +# RdSAP 10 §3.4 conversion of external to internal dimensions (#1669) +# --------------------------------------------------------------------------- + +# `measurement_type` lodged value: 1 = internal dimensions, 2 = external. +_EXTERNAL_MEASUREMENT: Final[int] = 2 + + +def _table_3_row(*vals: int) -> Dict[str, int]: + """Expand a Table 3 wall-thickness row's 10 spec columns to the 13 age + bands: bands I/J/K share the 9th column and L/M share the 10th.""" + a, b, c, d, e, f, g, h, ijk, lm = vals + return { + "A": a, "B": b, "C": c, "D": d, "E": e, "F": f, "G": g, "H": h, + "I": ijk, "J": ijk, "K": ijk, "L": lm, "M": lm, + } + + +# RdSAP 10 Table 3 (PDF p.19): default main-dwelling wall thickness (mm) by wall +# construction row and age band, used only when the thickness is not lodged. +_TABLE_3_WALL_THICKNESS_MM: Final[Dict[str, Dict[str, int]]] = { + "solid": _table_3_row(220, 220, 220, 220, 240, 250, 270, 270, 300, 300), + "cavity": _table_3_row(250, 250, 250, 250, 250, 260, 270, 270, 300, 300), + "timber": _table_3_row(150, 150, 150, 250, 270, 270, 270, 270, 300, 300), + "cob": _table_3_row(540, 540, 540, 540, 540, 540, 560, 560, 590, 590), + "system": _table_3_row(250, 250, 250, 250, 250, 300, 300, 300, 300, 300), +} +# `wall_construction` code → Table 3 row. Table 3 has no stone row (stone is +# normally measured); stone (1/2) and any non-masonry/unknown fall back to the +# solid-masonry column. +_WALL_CONSTRUCTION_TO_TABLE_3_ROW: Final[Dict[int, str]] = { + 1: "solid", 2: "solid", 3: "solid", # stone/granite, sandstone, solid brick + 4: "cavity", 5: "timber", 6: "system", 7: "cob", +} + + +def _main_wall_thickness_m(epc: EpcPropertyData) -> Optional[float]: + """The main dwelling's wall thickness in metres (RdSAP 10 §3.4 `w`): the + lodged value if present, else the Table 3 default for the main part's wall + construction + age band. None when neither is derivable.""" + parts = epc.sap_building_parts + if not parts: + return None + main = parts[0] + if main.wall_thickness_mm is not None: + return main.wall_thickness_mm / 1000.0 + construction = main.wall_construction + row = ( + _WALL_CONSTRUCTION_TO_TABLE_3_ROW.get(construction, "solid") + if isinstance(construction, int) + else "solid" + ) + band = (main.construction_age_band or "").strip().upper()[:1] + mm = _TABLE_3_WALL_THICKNESS_MM[row].get(band) + return mm / 1000.0 if mm is not None else None + + +def _table_2_ratios( + built_form: Optional[str], area_ext: float, perim_ext: float, w: float +) -> tuple[float, float]: + """RdSAP 10 Table 2 (PDF p.18) whole-dwelling external→internal + (area_ratio, perimeter_ratio) by built form (1 detached, 2 semi, 3 + end-terrace, 4 mid-terrace, 5 enclosed end-terrace, 6 enclosed mid-terrace). + `area_ext`/`perim_ext` are the ground-floor FOOTPRINT (§3.4 Note 4: one ratio + for the whole dwelling). Returns (1, 1) — no conversion — for an unrecognised + form or a degenerate footprint.""" + if area_ext <= 0.0 or perim_ext <= 0.0: + return (1.0, 1.0) + bf = (built_form or "").strip() + if bf == "1": # detached + perim_int = perim_ext - 8.0 * w + area_int = area_ext - w * perim_int - 4.0 * w * w + elif bf in ("2", "3"): # semi-detached / end-terrace + if perim_ext**2 >= 8.0 * area_ext: + perim_int = perim_ext - 5.0 * w + a = 0.5 * (perim_ext - math.sqrt(perim_ext**2 - 8.0 * area_ext)) + area_int = area_ext - w * (perim_ext + 0.5 * a) + 3.0 * w * w + else: + perim_int = perim_ext - 3.0 * w + area_int = area_ext - w * perim_ext + 3.0 * w * w + elif bf == "4": # mid-terrace + perim_int = perim_ext - 2.0 * w + area_int = area_ext - w * (perim_ext + 2.0 * area_ext / perim_ext) + 2.0 * w * w + elif bf == "5": # enclosed end-terrace + perim_int = perim_ext - 3.0 * w + area_int = area_ext - 1.5 * w * perim_ext + 2.25 * w * w + elif bf == "6": # enclosed mid-terrace + perim_int = perim_ext - w + area_int = area_ext - w * (area_ext / perim_ext + 1.5 * perim_ext) + 1.5 * w * w + else: + return (1.0, 1.0) + if area_int <= 0.0 or perim_int <= 0.0: + return (1.0, 1.0) + return (area_int / area_ext, perim_int / perim_ext) + + +def _domain_total_floor_area_m2(building_parts: List[SapBuildingPart]) -> float: + """Sum the domain floor-dimension areas (plus each part's always-internal + room-in-roof floor area) — used to refresh `total_floor_area_m2` after a + §3.4 conversion, since `water_heating_from_cert` reads that scalar for + occupancy N.""" + total = 0.0 + for bp in building_parts: + for fd in bp.sap_floor_dimensions: + total += fd.total_floor_area_m2 or 0.0 + rir = bp.sap_room_in_roof + if rir is not None and rir.floor_area: + total += rir.floor_area + return total + + +def _with_internal_dimensions( + epc: EpcPropertyData, data: Dict[str, Any] +) -> EpcPropertyData: + """Plumb the lodged `measurement_type` onto `epc` and, when the horizontal + dimensions were measured externally (== 2), convert them to internal + dimensions per RdSAP 10 §3.4 + Table 2 before they reach the cascade (#1669). + + §3.4 Note 4: ONE whole-dwelling area/perimeter ratio (from the ground-floor + footprint) is applied to every storey of every part — not a per-storey + conversion, which over-corrects multi-part mid-terraces. Room-in-roof floor + area is always measured internally (§3.1) and is left untouched; party wall + lengths are reduced by 2w (Table 2 Note 5). Internal certs are unchanged. + + Scope: the conversion is applied only to the RdSAP-Schema reduced-data family + (17.0-21.x), where per-storey external dimensions are the lodgement standard + and the fix is corpus-validated (RdSAP-21.0.1). `measurement_type` is still + plumbed for every schema, but the legacy SAP-Schema-15.0/16.x certs — whose + per-storey areas do not reconcile to the lodged TFA — are left unconverted.""" + epc.measurement_type = data.get("measurement_type") + schema_type = data.get("schema_type", "") + if ( + epc.measurement_type != _EXTERNAL_MEASUREMENT + or not isinstance(schema_type, str) + or not schema_type.startswith("RdSAP-Schema-") + or not epc.sap_building_parts + ): + return epc + w = _main_wall_thickness_m(epc) + if w is None: + return epc + ground_levels = [ + fd.floor + for bp in epc.sap_building_parts + for fd in bp.sap_floor_dimensions + if fd.floor is not None + ] + if not ground_levels: + return epc + ground = min(ground_levels) + area_ext = sum( + fd.total_floor_area_m2 or 0.0 + for bp in epc.sap_building_parts + for fd in bp.sap_floor_dimensions + if fd.floor == ground + ) + perim_ext = sum( + fd.heat_loss_perimeter_m or 0.0 + for bp in epc.sap_building_parts + for fd in bp.sap_floor_dimensions + if fd.floor == ground + ) + area_ratio, perim_ratio = _table_2_ratios(epc.built_form, area_ext, perim_ext, w) + if area_ratio == 1.0 and perim_ratio == 1.0: + return epc + for bp in epc.sap_building_parts: + for fd in bp.sap_floor_dimensions: + fd.total_floor_area_m2 *= area_ratio + fd.heat_loss_perimeter_m *= perim_ratio + if fd.party_wall_length_m: + fd.party_wall_length_m = max(0.0, fd.party_wall_length_m - 2.0 * w) + epc.total_floor_area_m2 = _domain_total_floor_area_m2(epc.sap_building_parts) + return epc + + def _with_recorded_performance( epc: EpcPropertyData, data: Dict[str, Any] ) -> EpcPropertyData: diff --git a/tests/datatypes/epc/domain/test_mapper_external_measurement_conversion.py b/tests/datatypes/epc/domain/test_mapper_external_measurement_conversion.py new file mode 100644 index 000000000..0c6cb8d8c --- /dev/null +++ b/tests/datatypes/epc/domain/test_mapper_external_measurement_conversion.py @@ -0,0 +1,93 @@ +"""#1669 — when a cert lodges its horizontal dimensions measured externally +(`measurement_type == 2`), RdSAP 10 §3.4 + Table 2 (spec p.17-18) require the +external floor area and exposed perimeter to be converted to INTERNAL dimensions +before any SAP use. `measurement_type` was read nowhere, so external certs +carried a TFA ~11-15% too large — corrupting occupancy, reported floor area and +every per-m² output. The mapper must plumb `measurement_type` through and, for +external certs, apply the Table 2 whole-dwelling ratio (§3.4 Note 4) before the +values reach the cascade, while leaving internal (`== 1`) certs verbatim. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +from datatypes.epc.domain.mapper import EpcPropertyDataMapper +from domain.sap10_calculator.calculator import Sap10Calculator + +_CORPUS = Path("backend/epc_api/json_samples/RdSAP-Schema-21.0.1/corpus.jsonl") + + +def _corpus_cert(uprn: str) -> dict[str, Any]: + for line in _CORPUS.read_text().splitlines(): + if uprn in line: + cert: dict[str, Any] = json.loads(line) + return cert + raise AssertionError(f"uprn {uprn} not found in the RdSAP-21.0.1 corpus") + + +def _with_measurement_type(payload: dict[str, Any], value: int) -> dict[str, Any]: + twin = deepcopy(payload) + twin["measurement_type"] = value + return twin + + +def _mapped_tfa(payload: dict[str, Any]) -> float: + epc = EpcPropertyDataMapper.from_api_response(payload) + assert epc is not None + assert epc.total_floor_area_m2 is not None + return epc.total_floor_area_m2 + + +def test_measurement_type_is_plumbed_onto_the_domain_object() -> None: + # Arrange — cert 100091435353 lodges measurement_type 2; the field was + # dropped to None by every API mapper (`# What is this?`). + payload = _corpus_cert("100091435353") + + # Act + epc = EpcPropertyDataMapper.from_api_response(payload) + + # Assert + assert epc is not None + assert epc.measurement_type == 2 + + +def test_external_cert_tfa_is_converted_to_internal_dimensions() -> None: + # Arrange — 100091435353 (detached, w=270mm) lodges a 128.7 m^2 EXTERNAL + # floor-area sum; its lodged internal TFA is 112. + payload = _corpus_cert("100091435353") + + # Act + tfa = _mapped_tfa(payload) + + # Assert — Table 2 detached conversion recovers the internal TFA to <0.5 m^2, + # not the raw external 128.7. + assert abs(tfa - 112.0) <= 0.5 + + +def test_internal_cert_dimensions_are_left_verbatim() -> None: + # Arrange — the same cert forced to measurement_type 1 (internal); §3.4 does + # not apply, so the floor areas must be used exactly as lodged (128.7). + payload = _with_measurement_type(_corpus_cert("100091435353"), 1) + + # Act + tfa = _mapped_tfa(payload) + + # Assert — no conversion; the external sum is passed through unchanged. + assert abs(tfa - 128.7) <= 1e-6 + + +def test_external_conversion_moves_sap_toward_lodged() -> None: + # Arrange — 100091435353 over-rated at 66.23 on HEAD (external dims), lodged 65. + payload = _corpus_cert("100091435353") + epc = EpcPropertyDataMapper.from_api_response(payload) + assert epc is not None + + # Act + sap = Sap10Calculator().calculate(epc).sap_score_continuous + + # Assert — the internal-dimension conversion pulls it to ~65.33, toward lodged. + assert abs(sap - 65.33) <= 0.05