From c9fc2c6baf2ff5d6c2619d96c77e06231eebbcd5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 9 Jul 2026 18:30:08 +0000 Subject: [PATCH] Map SAP-Schema-15.0 certs (LIG-lodged, 2011-era reduced-field RdSAP shape) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task a40e71c4-fd56-4a33-8677-e8643e6a5bb4 (portfolio 824, scenario 1278) failed 2 of 29 properties with "Unsupported EPC schema: 'SAP-Schema-15.0'" (property_id 748445/748451). Traced to two real 2011 lodgements (schema_version "LIG-15.0", assessment_type "RdSAP") for UPRNs 100010359769/100010359788 — structurally the same reduced-field RdSAP-17.1 shape as the already-handled SAP-Schema-16.x family, one revision older. from_sap_schema_15_0 reuses _normalize_sap_schema_16_x and adds three schema-wide defaults this older generation never lodges at all (confirmed absent on both real certs, not just sparse on one): - door_count=1 (single external door, standard RdSAP assumption) - instantaneous_wwhrs=0/0/0 and has_fghrs="N" (predates WWHRS/FGHRS lodging — absence-means-none-fitted, same convention used elsewhere) - percent_draughtproofed=0 (mirrors the existing has_draught_lobby "assume none if unknown" convention in cert_to_inputs.py) - energy_rating_average=60 (inert metadata, never read by the calculator) Both real certs now map and run end-to-end through the SAP-10 engine, matching (or within 1 point of) their lodged SAP score. --- datatypes/epc/domain/mapper.py | 60 +++++ .../domain/tests/test_from_sap_schema_15_0.py | 67 +++++ .../fixtures/sap_15_0_uprn_100010359769.json | 255 ++++++++++++++++++ .../fixtures/sap_15_0_uprn_100010359788.json | 248 +++++++++++++++++ 4 files changed, 630 insertions(+) create mode 100644 datatypes/epc/domain/tests/test_from_sap_schema_15_0.py create mode 100644 datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359769.json create mode 100644 datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359788.json diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index e3cdde0be..33b3cd840 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -1101,6 +1101,62 @@ class EpcPropertyDataMapper: schema convention.""" return EpcPropertyDataMapper.from_sap_schema_17_1(schema) + @staticmethod + def from_sap_schema_15_0(data: Dict[str, Any]) -> EpcPropertyData: + """Map a reduced-field `SAP-Schema-15.0` cert. + + Structurally the same LIG-lodged reduced-field shape as the + `SAP-Schema-16.x` family (`schema_version: "LIG-15.0"`, `assessment_type: + "RdSAP"`) — see `from_sap_schema_16_2`. One further gap on top of that + family's normalisation: these 2011-era certs never lodge `door_count` + at all (confirmed on two real certs, UPRNs 100010359769/100010359788 — + it is absent from the payload entirely, not blank). Unlike a 16.x cert + that omits `door_count` on an otherwise-complete lodgement (a per-cert + data gap the mapper correctly fails loud on), the whole 15.0 schema + never carried the field — so default to 1 (a single external door, + the standard RdSAP assumption when unrecorded) rather than treating it + as insufficient data. + + These certs also predate WWHRS and FGHRS lodging altogether — + `sap_heating` has no `wwhrs` key for `_normalize_sap_schema_16_x` to + rename, and `main_heating_details[]` never carries `has_fghrs`. Both + default to "not present" ("N" / all-zero counts), the same + absence-means-none-fitted convention used elsewhere in this mapper + (e.g. the full-SAP mappers' bare `InstantaneousWwhrs()`). + """ + from datatypes.epc.schema.rdsap_schema_17_1 import RdSapSchema17_1 + + normalized = _normalize_sap_schema_16_x(data) + normalized.setdefault("door_count", 1) + # `energy_rating_average` is the fixed national-average marker (60 on + # every fixture in this repo, `RdSapSchema17_1`-required, inert + # metadata never read by the SAP calculator) — these certs omit it. + normalized.setdefault("energy_rating_average", 60) + # Draught-proofing extent is unrecorded on these certs; 0% is the same + # "assume none if unknown" convention the cascade already applies to + # `has_draught_lobby` (see `_has_draught_lobby` in cert_to_inputs.py) — + # conservative (no draught-proofing credit), not a guessed value. + normalized.setdefault("percent_draughtproofed", 0) + sap_heating: Any = normalized.get("sap_heating") + if isinstance(sap_heating, dict): + heating: Dict[str, Any] = cast(Dict[str, Any], sap_heating) + heating.setdefault( + "instantaneous_wwhrs", + { + "rooms_with_bath_and_or_shower": 0, + "rooms_with_mixer_shower_no_bath": 0, + "rooms_with_bath_and_mixer_shower": 0, + }, + ) + main_heating_details: Any = heating.get("main_heating_details") or [] + for mh in cast(List[Any], main_heating_details): + if isinstance(mh, dict): + cast(Dict[str, Any], mh).setdefault("has_fghrs", "N") + + return EpcPropertyDataMapper.from_rdsap_schema_17_1( + from_dict(RdSapSchema17_1, normalized) + ) + @staticmethod def from_sap_schema_16_0(data: Dict[str, Any]) -> EpcPropertyData: """Map a reduced-field `SAP-Schema-16.0` cert (see `from_sap_schema_16_2`).""" @@ -2974,6 +3030,10 @@ class EpcPropertyDataMapper: mapped = EpcPropertyDataMapper.from_sap_schema_19_1_0( from_dict(SapSchema17_1, data) ) + # SAP-Schema-15.0: same reduced-field (RdSAP-shaped) family as 16.x, one + # revision older — see `from_sap_schema_15_0`. + elif schema == "SAP-Schema-15.0": + mapped = EpcPropertyDataMapper.from_sap_schema_15_0(data) # SAP-Schema-16.x family: reduced-field (RdSAP-shaped) certs. Each gets a # dedicated mapper that normalises onto the RdSAP-17.1 shape via the # shared `_normalize_sap_schema_16_x` and reuses `from_rdsap_schema_17_1`. diff --git a/datatypes/epc/domain/tests/test_from_sap_schema_15_0.py b/datatypes/epc/domain/tests/test_from_sap_schema_15_0.py new file mode 100644 index 000000000..570b84df9 --- /dev/null +++ b/datatypes/epc/domain/tests/test_from_sap_schema_15_0.py @@ -0,0 +1,67 @@ +"""SAP-Schema-15.0 — the same LIG-lodged reduced-field RdSAP-17.1 shape as +`SAP-Schema-16.x` (`schema_version: "LIG-15.0"`), one revision older, from a +2011 lodgement generation that predates `door_count`, WWHRS, and FGHRS +lodging entirely (unlike a 16.x cert omitting one of those fields, which is a +per-cert data gap, this is schema-wide — see `from_sap_schema_15_0`). + +`from_api_response` normalises it onto the RdSAP-17.1 shape via +`_normalize_sap_schema_16_x` plus the three extra defaults, then reuses that +mapper. Regression for the task that first surfaced this gap +(a40e71c4-fd56-4a33-8677-e8643e6a5bb4, portfolio 824 / scenario 1278, +property_id 748445 / 748451 — "Unsupported EPC schema: 'SAP-Schema-15.0'"). +""" + +from typing import Any, Dict +import json +import os + +from datatypes.epc.domain.epc_property_data import EpcPropertyData +from datatypes.epc.domain.mapper import EpcPropertyDataMapper + +FIXTURES = os.path.join(os.path.dirname(__file__), "../../schema/tests/fixtures") + + +def load(filename: str) -> Dict[str, Any]: + with open(os.path.join(FIXTURES, filename)) as f: + return json.load(f) # type: ignore[no-any-return] + + +class TestFromSapSchema15_0: + def test_dispatches_and_maps_via_rdsap_17_1(self) -> None: + # Arrange / Act — the raw 15.0 doc must dispatch (no "Unsupported + # schema") and map the 16.x-family seams plus the three schema-wide + # gaps unique to 15.0 (door_count, instantaneous_wwhrs, has_fghrs). + epc = EpcPropertyDataMapper.from_api_response( + load("sap_15_0_uprn_100010359769.json") + ) + + # Assert + assert isinstance(epc, EpcPropertyData) + assert epc.uprn == 100010359769 + assert epc.dwelling_type == "Mid-terrace house" + assert epc.door_count == 1 + mh = epc.sap_heating.main_heating_details[0] + assert mh.main_fuel_type == 26 # mains gas + + def test_produces_a_sap_score(self) -> None: + # The mapped cert must run end-to-end through the SAP-10 engine. + from domain.sap10_calculator.calculator import Sap10Calculator + + epc = EpcPropertyDataMapper.from_api_response( + load("sap_15_0_uprn_100010359769.json") + ) + result = Sap10Calculator().calculate(epc) + # Lodged 68; engine also produces 68. + assert result.sap_score == 68 + + def test_second_property_produces_a_sap_score(self) -> None: + # Second real cert from the same failing task, confirms the fix isn't + # a one-cert coincidence. + from domain.sap10_calculator.calculator import Sap10Calculator + + epc = EpcPropertyDataMapper.from_api_response( + load("sap_15_0_uprn_100010359788.json") + ) + result = Sap10Calculator().calculate(epc) + # Lodged 73; engine produces 72 (Elmhurst validation not yet run). + assert result.sap_score == 72 diff --git a/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359769.json b/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359769.json new file mode 100644 index 000000000..572bbb0e3 --- /dev/null +++ b/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359769.json @@ -0,0 +1,255 @@ +{ + "uprn": 100010359769, + "roofs": [ + { + "description": "Pitched, 250 mm loft insulation", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + }, + { + "description": "Flat, limited insulation (assumed)", + "energy_efficiency_rating": 1, + "environmental_efficiency_rating": 1 + } + ], + "walls": [ + { + "description": "Cavity wall, filled cavity", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "floors": [ + { + "description": "Suspended, no insulation (assumed)", + "energy_efficiency_rating": 0, + "environmental_efficiency_rating": 0 + }, + { + "description": "Solid, no insulation (assumed)", + "energy_efficiency_rating": 0, + "environmental_efficiency_rating": 0 + } + ], + "status": "entered", + "windows": [ + { + "description": "Fully double glazed", + "energy_efficiency_rating": 3, + "environmental_efficiency_rating": 3 + } + ], + "lighting": { + "description": "Low energy lighting in 91% of fixed outlets", + "energy_efficiency_rating": 5, + "environmental_efficiency_rating": 5 + }, + "postcode": "BB12 7BY", + "hot_water": { + "description": "From main system", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + }, + "post_town": "BURNLEY", + "built_form": 4, + "created_at": "2011-07-25 16:35:41", + "glazed_area": 1, + "region_code": 19, + "report_type": 2, + "sap_heating": { + "cylinder_size": 1, + "water_heating_code": 901, + "water_heating_fuel": 26, + "secondary_fuel_type": 29, + "main_heating_details": [ + { + "main_fuel_type": 26, + "boiler_flue_type": 2, + "fan_flue_present": "Y", + "heat_emitter_type": 1, + "boiler_index_number": 8358, + "main_heating_number": 1, + "main_heating_control": 2106, + "main_heating_category": 2, + "main_heating_fraction": 1, + "main_heating_data_source": 1 + } + ], + "secondary_heating_type": 691, + "has_fixed_air_conditioning": "false" + }, + "sap_version": 9.9, + "schema_type": "SAP-Schema-15.0", + "uprn_source": "Energy Assessor", + "country_code": "EAW", + "main_heating": [ + { + "description": "Boiler and radiators, mains gas", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "dwelling_type": "Mid-terrace house", + "language_code": 1, + "property_type": 0, + "address_line_1": "3, Warwick Drive", + "address_line_2": "Padiham", + "schema_version": "LIG-15.0", + "assessment_type": "RdSAP", + "completion_date": "2011-07-25", + "inspection_date": "2011-07-25", + "extensions_count": 1, + "measurement_type": 1, + "total_floor_area": 76, + "transaction_type": 3, + "conservatory_type": 1, + "heated_room_count": 5, + "registration_date": "2011-07-25", + "restricted_access": 0, + "sap_energy_source": { + "main_gas": "Y", + "meter_type": 2, + "photovoltaic_supply": { + "percent_roof_area": 0 + }, + "wind_turbines_count": 0, + "wind_turbines_terrain_type": 2 + }, + "secondary_heating": { + "description": "Room heaters, electric", + "energy_efficiency_rating": 0, + "environmental_efficiency_rating": 0 + }, + "sap_building_parts": [ + { + "identifier": "Main Dwelling", + "floor_heat_loss": 7, + "roof_construction": 4, + "wall_construction": 4, + "building_part_number": 1, + "sap_floor_dimensions": [ + { + "floor": 0, + "room_height": 2.58, + "floor_insulation": 1, + "total_floor_area": 35.59, + "floor_construction": 2, + "heat_loss_perimeter": 9 + }, + { + "floor": 1, + "room_height": 2.43, + "total_floor_area": 34.75, + "heat_loss_perimeter": 10.86 + } + ], + "wall_insulation_type": 2, + "construction_age_band": "C", + "roof_insulation_location": 2, + "roof_insulation_thickness": "250mm" + }, + { + "identifier": "Extension 1", + "floor_heat_loss": 7, + "roof_construction": 1, + "wall_construction": 4, + "building_part_number": 2, + "sap_floor_dimensions": [ + { + "floor": 0, + "room_height": 2.33, + "floor_insulation": 1, + "total_floor_area": 5.37, + "floor_construction": 1, + "heat_loss_perimeter": 4.66 + } + ], + "wall_insulation_type": 2, + "construction_age_band": "E", + "roof_insulation_location": 4, + "roof_insulation_thickness": "NI" + } + ], + "low_energy_lighting": 91, + "solar_water_heating": "N", + "bedf_revision_number": 311, + "habitable_room_count": 5, + "heating_cost_current": 501, + "co2_emissions_current": 2.8, + "energy_rating_current": 68, + "lighting_cost_current": 45, + "main_heating_controls": [ + { + "description": "Programmer, room thermostat and TRVs", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "multiple_glazing_type": 3, + "open_fireplaces_count": 0, + "has_hot_water_cylinder": "false", + "heating_cost_potential": 471, + "hot_water_cost_current": 95, + "mechanical_ventilation": 0, + "suggested_improvements": [ + { + "sequence": 1, + "typical_saving": 41, + "indicative_cost": "\u00a31,500 - \u00a33,500", + "improvement_type": "I", + "improvement_details": { + "improvement_number": 20 + }, + "improvement_category": 2, + "energy_performance_rating": 70, + "environmental_impact_rating": 71 + }, + { + "sequence": 2, + "typical_saving": 24, + "indicative_cost": "\u00a34,000 - \u00a36,000", + "improvement_type": "N", + "improvement_details": { + "improvement_number": 19 + }, + "improvement_category": 3, + "energy_performance_rating": 71, + "environmental_impact_rating": 73 + }, + { + "sequence": 3, + "typical_saving": 214, + "indicative_cost": "\u00a311,000 - \u00a320,000", + "improvement_type": "U", + "improvement_details": { + "improvement_number": 34 + }, + "improvement_category": 3, + "energy_performance_rating": 82, + "environmental_impact_rating": 83 + } + ], + "co2_emissions_potential": 2.6, + "energy_rating_potential": 70, + "lighting_cost_potential": 45, + "hot_water_cost_potential": 83, + "renewable_heat_incentive": { + "water_heating": 2071, + "space_heating_existing_dwelling": 6949, + "space_heating_with_loft_and_cavity_insulation": 6949 + }, + "seller_commission_report": "Y", + "energy_consumption_current": 196, + "has_fixed_air_conditioning": "false", + "multiple_glazed_proportion": 100, + "calculation_software_version": "19.05r12", + "energy_consumption_potential": 180, + "environmental_impact_current": 69, + "fixed_lighting_outlets_count": 11, + "current_energy_efficiency_band": "D", + "environmental_impact_potential": 71, + "has_heated_separate_conservatory": "false", + "potential_energy_efficiency_band": "C", + "co2_emissions_current_per_floor_area": 37, + "low_energy_fixed_lighting_outlets_count": 10 +} \ No newline at end of file diff --git a/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359788.json b/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359788.json new file mode 100644 index 000000000..1a80fde24 --- /dev/null +++ b/datatypes/epc/schema/tests/fixtures/sap_15_0_uprn_100010359788.json @@ -0,0 +1,248 @@ +{ + "uprn": 100010359788, + "roofs": [ + { + "description": "Pitched, 150 mm loft insulation", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + }, + { + "description": "Flat, limited insulation (assumed)", + "energy_efficiency_rating": 2, + "environmental_efficiency_rating": 2 + } + ], + "walls": [ + { + "description": "Cavity wall, filled cavity", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "floors": [ + { + "description": "Solid, no insulation (assumed)", + "energy_efficiency_rating": 0, + "environmental_efficiency_rating": 0 + } + ], + "status": "entered", + "windows": [ + { + "description": "Fully double glazed", + "energy_efficiency_rating": 3, + "environmental_efficiency_rating": 3 + } + ], + "lighting": { + "description": "Low energy lighting in 89% of fixed outlets", + "energy_efficiency_rating": 5, + "environmental_efficiency_rating": 5 + }, + "postcode": "BB12 7BY", + "hot_water": { + "description": "From main system", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + }, + "post_town": "BURNLEY", + "built_form": 4, + "created_at": "2011-08-04 07:03:30", + "glazed_area": 1, + "region_code": 19, + "report_type": 2, + "sap_heating": { + "cylinder_size": 1, + "water_heating_code": 901, + "water_heating_fuel": 26, + "main_heating_details": [ + { + "main_fuel_type": 26, + "boiler_flue_type": 2, + "fan_flue_present": "Y", + "heat_emitter_type": 1, + "boiler_index_number": 8358, + "main_heating_number": 1, + "main_heating_control": 2106, + "main_heating_category": 2, + "main_heating_fraction": 1, + "main_heating_data_source": 1 + } + ], + "has_fixed_air_conditioning": "false" + }, + "sap_version": 9.9, + "schema_type": "SAP-Schema-15.0", + "uprn_source": "Energy Assessor", + "country_code": "EAW", + "main_heating": [ + { + "description": "Boiler and radiators, mains gas", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "dwelling_type": "Mid-terrace house", + "language_code": 1, + "property_type": 0, + "address_line_1": "22, Warwick Drive", + "address_line_2": "Padiham", + "schema_version": "LIG-15.0", + "assessment_type": "RdSAP", + "completion_date": "2011-08-04", + "inspection_date": "2011-08-04", + "extensions_count": 1, + "measurement_type": 1, + "total_floor_area": 77, + "transaction_type": 3, + "conservatory_type": 1, + "heated_room_count": 5, + "registration_date": "2011-08-04", + "restricted_access": 0, + "sap_energy_source": { + "main_gas": "Y", + "meter_type": 2, + "photovoltaic_supply": { + "percent_roof_area": 0 + }, + "wind_turbines_count": 0, + "wind_turbines_terrain_type": 2 + }, + "secondary_heating": { + "description": "None", + "energy_efficiency_rating": 0, + "environmental_efficiency_rating": 0 + }, + "sap_building_parts": [ + { + "identifier": "Main Dwelling", + "floor_heat_loss": 7, + "roof_construction": 4, + "wall_construction": 4, + "building_part_number": 1, + "sap_floor_dimensions": [ + { + "floor": 0, + "room_height": 2.35, + "floor_insulation": 1, + "total_floor_area": 35, + "floor_construction": 1, + "heat_loss_perimeter": 8.3 + }, + { + "floor": 1, + "room_height": 2.4, + "total_floor_area": 35, + "heat_loss_perimeter": 11 + } + ], + "wall_insulation_type": 2, + "construction_age_band": "C", + "roof_insulation_location": 2, + "roof_insulation_thickness": "150mm" + }, + { + "identifier": "Extension 1", + "floor_heat_loss": 7, + "roof_construction": 1, + "wall_construction": 4, + "building_part_number": 2, + "sap_floor_dimensions": [ + { + "floor": 0, + "room_height": 2, + "floor_insulation": 1, + "total_floor_area": 6.63, + "floor_construction": 1, + "heat_loss_perimeter": 5.16 + } + ], + "wall_insulation_type": 2, + "construction_age_band": "F", + "roof_insulation_location": 4, + "roof_insulation_thickness": "NI" + } + ], + "low_energy_lighting": 89, + "solar_water_heating": "N", + "bedf_revision_number": 312, + "habitable_room_count": 5, + "heating_cost_current": 409, + "co2_emissions_current": 2.4, + "energy_rating_current": 73, + "lighting_cost_current": 46, + "main_heating_controls": [ + { + "description": "Programmer, room thermostat and TRVs", + "energy_efficiency_rating": 4, + "environmental_efficiency_rating": 4 + } + ], + "multiple_glazing_type": 1, + "open_fireplaces_count": 0, + "has_hot_water_cylinder": "false", + "heating_cost_potential": 379, + "hot_water_cost_current": 95, + "mechanical_ventilation": 0, + "suggested_improvements": [ + { + "sequence": 1, + "typical_saving": 41, + "indicative_cost": "\u00a31,500 - \u00a33,500", + "improvement_type": "I", + "improvement_details": { + "improvement_number": 20 + }, + "improvement_category": 2, + "energy_performance_rating": 75, + "environmental_impact_rating": 76 + }, + { + "sequence": 2, + "typical_saving": 25, + "indicative_cost": "\u00a34,000 - \u00a36,000", + "improvement_type": "N", + "improvement_details": { + "improvement_number": 19 + }, + "improvement_category": 3, + "energy_performance_rating": 76, + "environmental_impact_rating": 77 + }, + { + "sequence": 3, + "typical_saving": 214, + "indicative_cost": "\u00a311,000 - \u00a320,000", + "improvement_type": "U", + "improvement_details": { + "improvement_number": 34 + }, + "improvement_category": 3, + "energy_performance_rating": 86, + "environmental_impact_rating": 87 + } + ], + "co2_emissions_potential": 2.2, + "energy_rating_potential": 75, + "lighting_cost_potential": 46, + "hot_water_cost_potential": 84, + "renewable_heat_incentive": { + "water_heating": 2080, + "space_heating_existing_dwelling": 6207, + "space_heating_with_loft_and_cavity_insulation": 6207 + }, + "seller_commission_report": "Y", + "energy_consumption_current": 166, + "has_fixed_air_conditioning": "false", + "multiple_glazed_proportion": 100, + "calculation_software_version": "19.05r15", + "energy_consumption_potential": 150, + "environmental_impact_current": 73, + "fixed_lighting_outlets_count": 9, + "current_energy_efficiency_band": "C", + "environmental_impact_potential": 76, + "has_heated_separate_conservatory": "false", + "potential_energy_efficiency_band": "C", + "co2_emissions_current_per_floor_area": 32, + "low_energy_fixed_lighting_outlets_count": 8 +} \ No newline at end of file