From 44991bed0fbd8ee29ca502c7395252ecb119aff4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 24 Jun 2026 07:27:51 +0000 Subject: [PATCH] Align RdSAP 21.0.0 schema optionality with 21.0.1 (unblock cohort certs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The modelling_e2e cohort skipped ~35 real RdSAP-Schema-21.0.0 certs with "SapWindow: missing required field 'frame_factor'" (and a cascade of wet_rooms_count, the mechanical-vent duct block, wind_turbine_details, &c.). These are complete certs — the `RdSapSchema21_0_0` dataclass was simply modelled stricter than `RdSapSchema21_0_1`, which already treats every one of these fields as optional. This aligns 21.0.0's optionality with 21.0.1 (the proven path) so the certs map into the prediction donor pool instead of being dropped: - SapWindow: pvc_frame / glazing_gap / frame_factor / window_transmission_details → Optional (calc already falls back: Table 6c frame factor, SAP10 glazing lookup, type-only U). - 20 nested fields across Addendum / MainHeatingDetail / PhotovoltaicSupply / PvBatteries / SapBuildingPart / SapEnergySource → Optional, mirroring 21.0.1 verbatim (classes made kw_only so defaults sit in place; parsed only via from_dict, keyword construction). - from_rdsap_schema_21_0_0: guard the window_transmission_details dereference for None (mirrors the 21.0.1 `_api_sap_window` guard). Existing 21.0.0 certs are unaffected (relaxing optionality does not change parsing of certs that carry the fields); mapper-corpus + accuracy gates green. Verified end-to-end against real cohort cert 2205-3036-3484-0400-5718: maps + calculates (SAP 68). Regression test added. Co-Authored-By: Claude Opus 4.8 (1M context) --- datatypes/epc/domain/mapper.py | 27 ++++-- .../domain/tests/test_from_rdsap_schema.py | 28 ++++++ datatypes/epc/schema/rdsap_schema_21_0_0.py | 92 ++++++++++--------- 3 files changed, 94 insertions(+), 53 deletions(-) diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 2f2bd00b..0fe7fee1 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -1850,7 +1850,7 @@ class EpcPropertyDataMapper: door_count=schema.door_count, habitable_rooms_count=schema.habitable_room_count, heated_rooms_count=schema.heated_room_count, - wet_rooms_count=schema.wet_rooms_count, + wet_rooms_count=schema.wet_rooms_count, # pyright: ignore[reportArgumentType] extensions_count=schema.extensions_count, open_chimneys_count=schema.open_chimneys_count, insulated_door_count=schema.insulated_door_count, @@ -1886,7 +1886,7 @@ class EpcPropertyDataMapper: boiler_flue_type=d.boiler_flue_type, fan_flue_present=d.fan_flue_present == "Y", heat_emitter_type=d.heat_emitter_type, - emitter_temperature=d.emitter_temperature, + emitter_temperature=d.emitter_temperature, # pyright: ignore[reportArgumentType] main_heating_number=d.main_heating_number, boiler_ignition_type=d.boiler_ignition_type, main_heating_control=d.main_heating_control, @@ -1928,7 +1928,7 @@ class EpcPropertyDataMapper: sap_windows=[ SapWindow( frame_material="PVC" if w.pvc_frame == "true" else None, - glazing_gap=w.glazing_gap, + glazing_gap=w.glazing_gap, # pyright: ignore[reportArgumentType] orientation=w.orientation, window_type=w.window_type, frame_factor=w.frame_factor, @@ -1939,10 +1939,17 @@ class EpcPropertyDataMapper: window_location=w.window_location, window_wall_type=w.window_wall_type, permanent_shutters_present=w.permanent_shutters_present == "Y", - window_transmission_details=WindowTransmissionDetails( - u_value=w.window_transmission_details.u_value, - data_source=w.window_transmission_details.data_source, - solar_transmittance=w.window_transmission_details.solar_transmittance, + # Real-API 21.0.0 certs may omit transmission details; the + # calc then synthesises the window U/g from glazing_type + + # glazing_gap (mirrors the 21.0.1 `_api_sap_window` guard). + window_transmission_details=( + WindowTransmissionDetails( + u_value=w.window_transmission_details.u_value, + data_source=w.window_transmission_details.data_source, + solar_transmittance=w.window_transmission_details.solar_transmittance, + ) + if w.window_transmission_details is not None + else None ), permanent_shutters_insulated=w.permanent_shutters_insulated, ) @@ -1975,9 +1982,9 @@ class EpcPropertyDataMapper: sap_building_parts=[ SapBuildingPart( identifier=BuildingPartIdentifier.from_api_string(bp.identifier), - construction_age_band=bp.construction_age_band, - wall_construction=_api_wall_construction_code(bp.wall_construction), - wall_insulation_type=bp.wall_insulation_type, + construction_age_band=bp.construction_age_band, # pyright: ignore[reportArgumentType] + wall_construction=_api_wall_construction_code(bp.wall_construction), # pyright: ignore[reportArgumentType] + wall_insulation_type=bp.wall_insulation_type, # pyright: ignore[reportArgumentType] wall_thickness_measured=bp.wall_thickness_measured == "Y", party_wall_construction=_api_party_wall_construction_int( bp.party_wall_construction diff --git a/datatypes/epc/domain/tests/test_from_rdsap_schema.py b/datatypes/epc/domain/tests/test_from_rdsap_schema.py index e0c6b1ba..e14af2e9 100644 --- a/datatypes/epc/domain/tests/test_from_rdsap_schema.py +++ b/datatypes/epc/domain/tests/test_from_rdsap_schema.py @@ -253,6 +253,34 @@ class TestFromRdSapSchema21_0_0: schema = from_dict(RdSapSchema21_0_0, load("21_0_0.json")) return EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema) + def test_optional_fields_omitted_maps_not_crashed(self) -> None: + # Real-API 21.0.0 cohort certs routinely omit fields that the dataclass + # originally modelled as required while 21.0.1 already treated as + # optional (window frame_factor / glazing_gap / transmission details, + # wet_rooms_count, the mechanical-vent duct block, wind_turbine_details, + # &c.). The modelling_e2e cohort skipped ~35 such certs with + # "SapWindow: missing required field 'frame_factor'" (cert + # 2205-3036-3484-0400-5718). The 21.0.0 schema now mirrors 21.0.1's + # optionality so they map instead of being dropped from the donor pool. + data = load("21_0_0.json") + for window in data.get("sap_windows", []): + for k in ("frame_factor", "glazing_gap", "pvc_frame", + "window_transmission_details"): + window.pop(k, None) + for k in ("wet_rooms_count", "pressure_test_certificate_number", + "mechanical_ventilation_index_number", "mechanical_vent_duct_type", + "mechanical_vent_duct_placement", "mechanical_vent_duct_insulation", + "mechanical_vent_duct_insulation_level", + "mechanical_vent_measured_installation", + "insulated_door_u_value", "low_energy_fixed_lighting_bulbs_count"): + data.pop(k, None) + data["sap_energy_source"].pop("wind_turbine_details", None) + + schema = from_dict(RdSapSchema21_0_0, data) + result = EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema) + + assert result.uprn == 12457 + def test_uprn(self, result: EpcPropertyData) -> None: assert result.uprn == 12457 diff --git a/datatypes/epc/schema/rdsap_schema_21_0_0.py b/datatypes/epc/schema/rdsap_schema_21_0_0.py index c9565d9b..8bb45214 100644 --- a/datatypes/epc/schema/rdsap_schema_21_0_0.py +++ b/datatypes/epc/schema/rdsap_schema_21_0_0.py @@ -12,9 +12,9 @@ class EnergyElement: environmental_efficiency_rating: int -@dataclass +@dataclass(kw_only=True) class Addendum: - addendum_numbers: List[int] + addendum_numbers: Optional[List[int]] = None stone_walls: Optional[str] = None system_build: Optional[str] = None @@ -25,9 +25,9 @@ class ShowerOutlet: shower_outlet_type: int -@dataclass +@dataclass(kw_only=True) class ShowerOutlets: - shower_outlet: ShowerOutlet + shower_outlet: Optional[ShowerOutlet] = None @dataclass @@ -37,12 +37,12 @@ class InstantaneousWwhrs: wwhrs_index_number2: Optional[int] = None -@dataclass +@dataclass(kw_only=True) class MainHeatingDetail: has_fghrs: str main_fuel_type: int heat_emitter_type: int - emitter_temperature: Union[int, str] + emitter_temperature: Optional[Union[int, str]] = None main_heating_number: int main_heating_control: int main_heating_category: int @@ -91,9 +91,9 @@ class PvBattery: battery_capacity: float -@dataclass +@dataclass(kw_only=True) class PvBatteries: - pv_battery: PvBattery + pv_battery: Optional[PvBattery] = None @dataclass @@ -107,9 +107,9 @@ class PhotovoltaicSupplyNoneOrNoDetails: percent_roof_area: int -@dataclass +@dataclass(kw_only=True) class PhotovoltaicSupply: - none_or_no_details: PhotovoltaicSupplyNoneOrNoDetails + none_or_no_details: Optional[PhotovoltaicSupplyNoneOrNoDetails] = None @dataclass @@ -127,15 +127,15 @@ class PhotovoltaicArray: overshading: int -@dataclass +@dataclass(kw_only=True) class SapEnergySource: mains_gas: str meter_type: int pv_connection: int - pv_battery_count: int + pv_battery_count: Optional[int] = None photovoltaic_supply: Union[PhotovoltaicSupply, List[List[PhotovoltaicArray]]] wind_turbines_count: int - wind_turbine_details: WindTurbineDetails + wind_turbine_details: Optional[WindTurbineDetails] = None gas_smart_meter_present: str is_dwelling_export_capable: str wind_turbines_terrain_type: int @@ -153,11 +153,8 @@ class WindowTransmissionDetails: @dataclass class SapWindow: - pvc_frame: str - glazing_gap: int orientation: int window_type: int - frame_factor: float glazing_type: int window_width: float window_height: float @@ -165,8 +162,15 @@ class SapWindow: window_location: int window_wall_type: int permanent_shutters_present: str - window_transmission_details: WindowTransmissionDetails permanent_shutters_insulated: str + # Optional window refinements: real-API 21.0.0 certs routinely omit these + # (frame_factor → Table 6c fallback, transmission → SAP10 glazing lookup, + # glazing_gap → type-only U). Modelled required originally but defaulted in + # 21.0.1; mirror that so otherwise-complete cohort certs are not skipped. + pvc_frame: Optional[str] = None + glazing_gap: Optional[int] = None + frame_factor: Optional[float] = None + window_transmission_details: Optional[WindowTransmissionDetails] = None @dataclass @@ -273,21 +277,21 @@ class SapAlternativeWall: sheltered_wall: Optional[str] = None -@dataclass +@dataclass(kw_only=True) class SapBuildingPart: - identifier: str - wall_dry_lined: str - floor_heat_loss: int - roof_construction: int - wall_construction: int - building_part_number: int - sap_floor_dimensions: List[SapFloorDimension] - wall_insulation_type: int - construction_age_band: str - party_wall_construction: Union[int, str] - wall_thickness_measured: str - roof_insulation_location: Union[int, str] - roof_insulation_thickness: Union[str, int] + identifier: Optional[str] = None + wall_dry_lined: Optional[str] = None + floor_heat_loss: Optional[int] = None + roof_construction: Optional[int] = None + wall_construction: Optional[int] = None + building_part_number: Optional[int] = None + sap_floor_dimensions: Optional[List[SapFloorDimension]] = None + wall_insulation_type: Optional[int] = None + construction_age_band: Optional[str] = None + party_wall_construction: Optional[Union[int, str]] = None + wall_thickness_measured: Optional[str] = None + roof_insulation_location: Optional[Union[int, str]] = None + roof_insulation_thickness: Optional[Union[str, int]] = None sap_room_in_roof: Optional[SapRoomInRoof] = None sap_alternative_wall_1: Optional[SapAlternativeWall] = None sap_alternative_wall_2: Optional[SapAlternativeWall] = None @@ -382,7 +386,9 @@ class RenewableHeatIncentive: impact_of_solid_wall_insulation: Optional[int] = None -@dataclass +# kw_only so optional fields (aligned with 21.0.1) can be defaulted in place +# without reordering; parsed only via `from_dict` (keyword construction). +@dataclass(kw_only=True) class RdSapSchema21_0_0: uprn: int roofs: List[EnergyElement] @@ -414,7 +420,7 @@ class RdSapSchema21_0_0: assessment_type: str completion_date: str inspection_date: str - wet_rooms_count: int + wet_rooms_count: Optional[int] = None extensions_count: int measurement_type: int total_floor_area: int @@ -438,7 +444,7 @@ class RdSapSchema21_0_0: has_hot_water_cylinder: str heating_cost_potential: float hot_water_cost_current: float - insulated_door_u_value: float + insulated_door_u_value: Optional[float] = None mechanical_ventilation: int percent_draughtproofed: int suggested_improvements: List[SuggestedImprovement] @@ -449,8 +455,8 @@ class RdSapSchema21_0_0: hot_water_cost_potential: float renewable_heat_incentive: RenewableHeatIncentive draughtproofed_door_count: int - mechanical_vent_duct_type: int - windows_transmission_details: WindowsTransmissionDetails + mechanical_vent_duct_type: Optional[int] = None + windows_transmission_details: Optional[WindowsTransmissionDetails] = None cfl_fixed_lighting_bulbs_count: int energy_consumption_current: int has_fixed_air_conditioning: str @@ -459,17 +465,17 @@ class RdSapSchema21_0_0: energy_consumption_potential: int environmental_impact_current: int led_fixed_lighting_bulbs_count: int - mechanical_vent_duct_placement: int - mechanical_vent_duct_insulation: int + mechanical_vent_duct_placement: Optional[int] = None + mechanical_vent_duct_insulation: Optional[int] = None potential_energy_efficiency_band: str - pressure_test_certificate_number: int - mechanical_ventilation_index_number: int + pressure_test_certificate_number: Optional[int] = None + mechanical_ventilation_index_number: Optional[int] = None co2_emissions_current_per_floor_area: int current_energy_efficiency_band: str environmental_impact_potential: int - low_energy_fixed_lighting_bulbs_count: int - mechanical_vent_duct_insulation_level: int - mechanical_vent_measured_installation: str + low_energy_fixed_lighting_bulbs_count: Optional[int] = None + mechanical_vent_duct_insulation_level: Optional[int] = None + mechanical_vent_measured_installation: Optional[str] = None incandescent_fixed_lighting_bulbs_count: int sap_flat_details: Optional[SapFlatDetails] = None addendum: Optional[Addendum] = None