diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 1658383f..afc98872 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -1850,13 +1850,15 @@ class EpcPropertyDataMapper: door_count=schema.door_count, habitable_rooms_count=schema.habitable_room_count, heated_rooms_count=schema.heated_room_count, - wet_rooms_count=schema.wet_rooms_count, + # Optional in 21.0.0 (real certs omit them); coalesce counts to 0 + # like every other mapper so None can't reach a NOT-NULL column. + wet_rooms_count=schema.wet_rooms_count or 0, extensions_count=schema.extensions_count, - open_chimneys_count=schema.open_chimneys_count, + open_chimneys_count=schema.open_chimneys_count or 0, insulated_door_count=schema.insulated_door_count, draughtproofed_door_count=schema.draughtproofed_door_count, - led_fixed_lighting_bulbs_count=schema.led_fixed_lighting_bulbs_count, - cfl_fixed_lighting_bulbs_count=schema.cfl_fixed_lighting_bulbs_count, + led_fixed_lighting_bulbs_count=schema.led_fixed_lighting_bulbs_count or 0, + cfl_fixed_lighting_bulbs_count=schema.cfl_fixed_lighting_bulbs_count or 0, incandescent_fixed_lighting_bulbs_count=schema.incandescent_fixed_lighting_bulbs_count, roofs=EpcPropertyDataMapper._map_energy_elements(schema.roofs), walls=EpcPropertyDataMapper._map_energy_elements(schema.walls), @@ -1928,7 +1930,7 @@ class EpcPropertyDataMapper: sap_windows=[ SapWindow( frame_material="PVC" if w.pvc_frame == "true" else None, - glazing_gap=w.glazing_gap, + glazing_gap=w.glazing_gap, # pyright: ignore[reportArgumentType] orientation=w.orientation, window_type=w.window_type, frame_factor=w.frame_factor, @@ -1939,10 +1941,17 @@ class EpcPropertyDataMapper: window_location=w.window_location, window_wall_type=w.window_wall_type, permanent_shutters_present=w.permanent_shutters_present == "Y", - window_transmission_details=WindowTransmissionDetails( - u_value=w.window_transmission_details.u_value, - data_source=w.window_transmission_details.data_source, - solar_transmittance=w.window_transmission_details.solar_transmittance, + # Real-API 21.0.0 certs may omit transmission details; the + # calc then synthesises the window U/g from glazing_type + + # glazing_gap (mirrors the 21.0.1 `_api_sap_window` guard). + window_transmission_details=( + WindowTransmissionDetails( + u_value=w.window_transmission_details.u_value, + data_source=w.window_transmission_details.data_source, + solar_transmittance=w.window_transmission_details.solar_transmittance, + ) + if w.window_transmission_details is not None + else None ), permanent_shutters_insulated=w.permanent_shutters_insulated, ) diff --git a/datatypes/epc/domain/tests/test_from_rdsap_schema.py b/datatypes/epc/domain/tests/test_from_rdsap_schema.py index 023f7c60..a2463eae 100644 --- a/datatypes/epc/domain/tests/test_from_rdsap_schema.py +++ b/datatypes/epc/domain/tests/test_from_rdsap_schema.py @@ -253,6 +253,42 @@ class TestFromRdSapSchema21_0_0: schema = from_dict(RdSapSchema21_0_0, load("21_0_0.json")) return EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema) + def test_optional_fields_omitted_maps_not_crashed(self) -> None: + # Real-API 21.0.0 cohort certs omit a set of fields that the dataclass + # modelled as required while 21.0.1 already treats as optional. The + # modelling_e2e cohort skipped 33 such certs (e.g. 2205-3036-3484-0400- + # 5718). This is the exact union of fields those certs omit — all are + # optional in 21.0.1; widening 21.0.0 to match maps them into the donor + # pool instead of dropping them. (Building-part fields are NOT widened: + # the only one those certs omit is roof_insulation_thickness.) + data = load("21_0_0.json") + for window in data.get("sap_windows", []): + for k in ("frame_factor", "glazing_gap", "pvc_frame", + "window_transmission_details"): + window.pop(k, None) + for k in ("wet_rooms_count", "open_chimneys_count", + "pressure_test_certificate_number", "windows_transmission_details", + "mechanical_ventilation_index_number", "mechanical_vent_duct_type", + "mechanical_vent_duct_placement", "mechanical_vent_duct_insulation", + "mechanical_vent_duct_insulation_level", + "mechanical_vent_measured_installation", "insulated_door_u_value", + "low_energy_fixed_lighting_bulbs_count", + "cfl_fixed_lighting_bulbs_count", "led_fixed_lighting_bulbs_count", + "suggested_improvements"): + data.pop(k, None) + data["sap_energy_source"].pop("wind_turbine_details", None) + data["sap_energy_source"].pop("pv_battery_count", None) + for bp in data.get("sap_building_parts", []): + bp.pop("roof_insulation_thickness", None) + + schema = from_dict(RdSapSchema21_0_0, data) + result = EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema) + + assert result.uprn == 12457 + # coalesced, not None → safe for the NOT-NULL epc_property column + assert result.wet_rooms_count == 0 + assert result.open_chimneys_count == 0 + def test_uprn(self, result: EpcPropertyData) -> None: assert result.uprn == 12457 diff --git a/datatypes/epc/schema/rdsap_schema_21_0_0.py b/datatypes/epc/schema/rdsap_schema_21_0_0.py index c9565d9b..2f5c8e69 100644 --- a/datatypes/epc/schema/rdsap_schema_21_0_0.py +++ b/datatypes/epc/schema/rdsap_schema_21_0_0.py @@ -14,7 +14,7 @@ class EnergyElement: @dataclass class Addendum: - addendum_numbers: List[int] + addendum_numbers: Optional[List[int]] = None stone_walls: Optional[str] = None system_build: Optional[str] = None @@ -109,7 +109,7 @@ class PhotovoltaicSupplyNoneOrNoDetails: @dataclass class PhotovoltaicSupply: - none_or_no_details: PhotovoltaicSupplyNoneOrNoDetails + none_or_no_details: Optional[PhotovoltaicSupplyNoneOrNoDetails] = None @dataclass @@ -127,15 +127,15 @@ class PhotovoltaicArray: overshading: int -@dataclass +@dataclass(kw_only=True) class SapEnergySource: mains_gas: str meter_type: int pv_connection: int - pv_battery_count: int + pv_battery_count: Optional[int] = None photovoltaic_supply: Union[PhotovoltaicSupply, List[List[PhotovoltaicArray]]] wind_turbines_count: int - wind_turbine_details: WindTurbineDetails + wind_turbine_details: Optional[WindTurbineDetails] = None gas_smart_meter_present: str is_dwelling_export_capable: str wind_turbines_terrain_type: int @@ -153,11 +153,8 @@ class WindowTransmissionDetails: @dataclass class SapWindow: - pvc_frame: str - glazing_gap: int orientation: int window_type: int - frame_factor: float glazing_type: int window_width: float window_height: float @@ -165,8 +162,15 @@ class SapWindow: window_location: int window_wall_type: int permanent_shutters_present: str - window_transmission_details: WindowTransmissionDetails permanent_shutters_insulated: str + # Optional window refinements: real-API 21.0.0 certs routinely omit these + # (frame_factor → Table 6c fallback, transmission → SAP10 glazing lookup, + # glazing_gap → type-only U). Modelled required originally but defaulted in + # 21.0.1; mirror that so otherwise-complete cohort certs are not skipped. + pvc_frame: Optional[str] = None + glazing_gap: Optional[int] = None + frame_factor: Optional[float] = None + window_transmission_details: Optional[WindowTransmissionDetails] = None @dataclass @@ -273,7 +277,7 @@ class SapAlternativeWall: sheltered_wall: Optional[str] = None -@dataclass +@dataclass(kw_only=True) class SapBuildingPart: identifier: str wall_dry_lined: str @@ -287,7 +291,7 @@ class SapBuildingPart: party_wall_construction: Union[int, str] wall_thickness_measured: str roof_insulation_location: Union[int, str] - roof_insulation_thickness: Union[str, int] + roof_insulation_thickness: Optional[Union[str, int]] = None sap_room_in_roof: Optional[SapRoomInRoof] = None sap_alternative_wall_1: Optional[SapAlternativeWall] = None sap_alternative_wall_2: Optional[SapAlternativeWall] = None @@ -382,7 +386,9 @@ class RenewableHeatIncentive: impact_of_solid_wall_insulation: Optional[int] = None -@dataclass +# kw_only so optional fields (aligned with 21.0.1) can be defaulted in place +# without reordering; parsed only via `from_dict` (keyword construction). +@dataclass(kw_only=True) class RdSapSchema21_0_0: uprn: int roofs: List[EnergyElement] @@ -414,7 +420,7 @@ class RdSapSchema21_0_0: assessment_type: str completion_date: str inspection_date: str - wet_rooms_count: int + wet_rooms_count: Optional[int] = None extensions_count: int measurement_type: int total_floor_area: int @@ -425,7 +431,7 @@ class RdSapSchema21_0_0: sap_energy_source: SapEnergySource secondary_heating: EnergyElement sap_building_parts: List[SapBuildingPart] - open_chimneys_count: int + open_chimneys_count: Optional[int] = None solar_water_heating: str habitable_room_count: int heating_cost_current: float @@ -438,10 +444,10 @@ class RdSapSchema21_0_0: has_hot_water_cylinder: str heating_cost_potential: float hot_water_cost_current: float - insulated_door_u_value: float + insulated_door_u_value: Optional[float] = None mechanical_ventilation: int percent_draughtproofed: int - suggested_improvements: List[SuggestedImprovement] + suggested_improvements: Optional[List[SuggestedImprovement]] = None co2_emissions_potential: float energy_rating_potential: int lighting_cost_potential: float @@ -449,27 +455,27 @@ class RdSapSchema21_0_0: hot_water_cost_potential: float renewable_heat_incentive: RenewableHeatIncentive draughtproofed_door_count: int - mechanical_vent_duct_type: int - windows_transmission_details: WindowsTransmissionDetails - cfl_fixed_lighting_bulbs_count: int + mechanical_vent_duct_type: Optional[int] = None + windows_transmission_details: Optional[WindowsTransmissionDetails] = None + cfl_fixed_lighting_bulbs_count: Optional[int] = None energy_consumption_current: int has_fixed_air_conditioning: str multiple_glazed_proportion: int calculation_software_version: str energy_consumption_potential: int environmental_impact_current: int - led_fixed_lighting_bulbs_count: int - mechanical_vent_duct_placement: int - mechanical_vent_duct_insulation: int + led_fixed_lighting_bulbs_count: Optional[int] = None + mechanical_vent_duct_placement: Optional[int] = None + mechanical_vent_duct_insulation: Optional[int] = None potential_energy_efficiency_band: str - pressure_test_certificate_number: int - mechanical_ventilation_index_number: int + pressure_test_certificate_number: Optional[int] = None + mechanical_ventilation_index_number: Optional[int] = None co2_emissions_current_per_floor_area: int current_energy_efficiency_band: str environmental_impact_potential: int - low_energy_fixed_lighting_bulbs_count: int - mechanical_vent_duct_insulation_level: int - mechanical_vent_measured_installation: str + low_energy_fixed_lighting_bulbs_count: Optional[int] = None + mechanical_vent_duct_insulation_level: Optional[int] = None + mechanical_vent_measured_installation: Optional[str] = None incandescent_fixed_lighting_bulbs_count: int sap_flat_details: Optional[SapFlatDetails] = None addendum: Optional[Addendum] = None