Merge pull request #1283 from Hestia-Homes/fix/rdsap-21-0-0-cohort-cert-skips

Align RdSAP 21.0.0 schema optionality with 21.0.1 (unblock ~35 cohort certs)

datatypes/epc/domain/mapper.py

@@ -1850,13 +1850,15 @@ class EpcPropertyDataMapper:
             door_count=schema.door_count,
             habitable_rooms_count=schema.habitable_room_count,
             heated_rooms_count=schema.heated_room_count,
-            wet_rooms_count=schema.wet_rooms_count,
+            # Optional in 21.0.0 (real certs omit them); coalesce counts to 0
+            # like every other mapper so None can't reach a NOT-NULL column.
+            wet_rooms_count=schema.wet_rooms_count or 0,
             extensions_count=schema.extensions_count,
-            open_chimneys_count=schema.open_chimneys_count,
+            open_chimneys_count=schema.open_chimneys_count or 0,
             insulated_door_count=schema.insulated_door_count,
             draughtproofed_door_count=schema.draughtproofed_door_count,
-            led_fixed_lighting_bulbs_count=schema.led_fixed_lighting_bulbs_count,
+            led_fixed_lighting_bulbs_count=schema.led_fixed_lighting_bulbs_count or 0,
-            cfl_fixed_lighting_bulbs_count=schema.cfl_fixed_lighting_bulbs_count,
+            cfl_fixed_lighting_bulbs_count=schema.cfl_fixed_lighting_bulbs_count or 0,
             incandescent_fixed_lighting_bulbs_count=schema.incandescent_fixed_lighting_bulbs_count,
             roofs=EpcPropertyDataMapper._map_energy_elements(schema.roofs),
             walls=EpcPropertyDataMapper._map_energy_elements(schema.walls),
@@ -1928,7 +1930,7 @@ class EpcPropertyDataMapper:
             sap_windows=[
                 SapWindow(
                     frame_material="PVC" if w.pvc_frame == "true" else None,
-                    glazing_gap=w.glazing_gap,
+                    glazing_gap=w.glazing_gap,  # pyright: ignore[reportArgumentType]
                     orientation=w.orientation,
                     window_type=w.window_type,
                     frame_factor=w.frame_factor,
@@ -1939,10 +1941,17 @@ class EpcPropertyDataMapper:
                     window_location=w.window_location,
                     window_wall_type=w.window_wall_type,
                     permanent_shutters_present=w.permanent_shutters_present == "Y",
-                    window_transmission_details=WindowTransmissionDetails(
+                    # Real-API 21.0.0 certs may omit transmission details; the
-                        u_value=w.window_transmission_details.u_value,
+                    # calc then synthesises the window U/g from glazing_type +
-                        data_source=w.window_transmission_details.data_source,
+                    # glazing_gap (mirrors the 21.0.1 `_api_sap_window` guard).
-                        solar_transmittance=w.window_transmission_details.solar_transmittance,
+                    window_transmission_details=(
+                        WindowTransmissionDetails(
+                            u_value=w.window_transmission_details.u_value,
+                            data_source=w.window_transmission_details.data_source,
+                            solar_transmittance=w.window_transmission_details.solar_transmittance,
+                        )
+                        if w.window_transmission_details is not None
+                        else None
                     ),
                     permanent_shutters_insulated=w.permanent_shutters_insulated,
                 )

datatypes/epc/domain/tests/test_from_rdsap_schema.py

@@ -253,6 +253,42 @@ class TestFromRdSapSchema21_0_0:
         schema = from_dict(RdSapSchema21_0_0, load("21_0_0.json"))
         return EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema)
 
+    def test_optional_fields_omitted_maps_not_crashed(self) -> None:
+        # Real-API 21.0.0 cohort certs omit a set of fields that the dataclass
+        # modelled as required while 21.0.1 already treats as optional. The
+        # modelling_e2e cohort skipped 33 such certs (e.g. 2205-3036-3484-0400-
+        # 5718). This is the exact union of fields those certs omit — all are
+        # optional in 21.0.1; widening 21.0.0 to match maps them into the donor
+        # pool instead of dropping them. (Building-part fields are NOT widened:
+        # the only one those certs omit is roof_insulation_thickness.)
+        data = load("21_0_0.json")
+        for window in data.get("sap_windows", []):
+            for k in ("frame_factor", "glazing_gap", "pvc_frame",
+                      "window_transmission_details"):
+                window.pop(k, None)
+        for k in ("wet_rooms_count", "open_chimneys_count",
+                  "pressure_test_certificate_number", "windows_transmission_details",
+                  "mechanical_ventilation_index_number", "mechanical_vent_duct_type",
+                  "mechanical_vent_duct_placement", "mechanical_vent_duct_insulation",
+                  "mechanical_vent_duct_insulation_level",
+                  "mechanical_vent_measured_installation", "insulated_door_u_value",
+                  "low_energy_fixed_lighting_bulbs_count",
+                  "cfl_fixed_lighting_bulbs_count", "led_fixed_lighting_bulbs_count",
+                  "suggested_improvements"):
+            data.pop(k, None)
+        data["sap_energy_source"].pop("wind_turbine_details", None)
+        data["sap_energy_source"].pop("pv_battery_count", None)
+        for bp in data.get("sap_building_parts", []):
+            bp.pop("roof_insulation_thickness", None)
+
+        schema = from_dict(RdSapSchema21_0_0, data)
+        result = EpcPropertyDataMapper.from_rdsap_schema_21_0_0(schema)
+
+        assert result.uprn == 12457
+        # coalesced, not None → safe for the NOT-NULL epc_property column
+        assert result.wet_rooms_count == 0
+        assert result.open_chimneys_count == 0
+
     def test_uprn(self, result: EpcPropertyData) -> None:
         assert result.uprn == 12457
 

datatypes/epc/schema/rdsap_schema_21_0_0.py

@@ -14,7 +14,7 @@ class EnergyElement:
 
 @dataclass
 class Addendum:
-    addendum_numbers: List[int]
+    addendum_numbers: Optional[List[int]] = None
     stone_walls: Optional[str] = None
     system_build: Optional[str] = None
 
@@ -109,7 +109,7 @@ class PhotovoltaicSupplyNoneOrNoDetails:
 
 @dataclass
 class PhotovoltaicSupply:
-    none_or_no_details: PhotovoltaicSupplyNoneOrNoDetails
+    none_or_no_details: Optional[PhotovoltaicSupplyNoneOrNoDetails] = None
 
 
 @dataclass
@@ -127,15 +127,15 @@ class PhotovoltaicArray:
     overshading: int
 
 
-@dataclass
+@dataclass(kw_only=True)
 class SapEnergySource:
     mains_gas: str
     meter_type: int
     pv_connection: int
-    pv_battery_count: int
+    pv_battery_count: Optional[int] = None
     photovoltaic_supply: Union[PhotovoltaicSupply, List[List[PhotovoltaicArray]]]
     wind_turbines_count: int
-    wind_turbine_details: WindTurbineDetails
+    wind_turbine_details: Optional[WindTurbineDetails] = None
     gas_smart_meter_present: str
     is_dwelling_export_capable: str
     wind_turbines_terrain_type: int
@@ -153,11 +153,8 @@ class WindowTransmissionDetails:
 
 @dataclass
 class SapWindow:
-    pvc_frame: str
-    glazing_gap: int
     orientation: int
     window_type: int
-    frame_factor: float
     glazing_type: int
     window_width: float
     window_height: float
@@ -165,8 +162,15 @@ class SapWindow:
     window_location: int
     window_wall_type: int
     permanent_shutters_present: str
-    window_transmission_details: WindowTransmissionDetails
     permanent_shutters_insulated: str
+    # Optional window refinements: real-API 21.0.0 certs routinely omit these
+    # (frame_factor → Table 6c fallback, transmission → SAP10 glazing lookup,
+    # glazing_gap → type-only U). Modelled required originally but defaulted in
+    # 21.0.1; mirror that so otherwise-complete cohort certs are not skipped.
+    pvc_frame: Optional[str] = None
+    glazing_gap: Optional[int] = None
+    frame_factor: Optional[float] = None
+    window_transmission_details: Optional[WindowTransmissionDetails] = None
 
 
 @dataclass
@@ -273,7 +277,7 @@ class SapAlternativeWall:
     sheltered_wall: Optional[str] = None
 
 
-@dataclass
+@dataclass(kw_only=True)
 class SapBuildingPart:
     identifier: str
     wall_dry_lined: str
@@ -287,7 +291,7 @@ class SapBuildingPart:
     party_wall_construction: Union[int, str]
     wall_thickness_measured: str
     roof_insulation_location: Union[int, str]
-    roof_insulation_thickness: Union[str, int]
+    roof_insulation_thickness: Optional[Union[str, int]] = None
     sap_room_in_roof: Optional[SapRoomInRoof] = None
     sap_alternative_wall_1: Optional[SapAlternativeWall] = None
     sap_alternative_wall_2: Optional[SapAlternativeWall] = None
@@ -382,7 +386,9 @@ class RenewableHeatIncentive:
     impact_of_solid_wall_insulation: Optional[int] = None
 
 
-@dataclass
+# kw_only so optional fields (aligned with 21.0.1) can be defaulted in place
+# without reordering; parsed only via `from_dict` (keyword construction).
+@dataclass(kw_only=True)
 class RdSapSchema21_0_0:
     uprn: int
     roofs: List[EnergyElement]
@@ -414,7 +420,7 @@ class RdSapSchema21_0_0:
     assessment_type: str
     completion_date: str
     inspection_date: str
-    wet_rooms_count: int
+    wet_rooms_count: Optional[int] = None
     extensions_count: int
     measurement_type: int
     total_floor_area: int
@@ -425,7 +431,7 @@ class RdSapSchema21_0_0:
     sap_energy_source: SapEnergySource
     secondary_heating: EnergyElement
     sap_building_parts: List[SapBuildingPart]
-    open_chimneys_count: int
+    open_chimneys_count: Optional[int] = None
     solar_water_heating: str
     habitable_room_count: int
     heating_cost_current: float
@@ -438,10 +444,10 @@ class RdSapSchema21_0_0:
     has_hot_water_cylinder: str
     heating_cost_potential: float
     hot_water_cost_current: float
-    insulated_door_u_value: float
+    insulated_door_u_value: Optional[float] = None
     mechanical_ventilation: int
     percent_draughtproofed: int
-    suggested_improvements: List[SuggestedImprovement]
+    suggested_improvements: Optional[List[SuggestedImprovement]] = None
     co2_emissions_potential: float
     energy_rating_potential: int
     lighting_cost_potential: float
@@ -449,27 +455,27 @@ class RdSapSchema21_0_0:
     hot_water_cost_potential: float
     renewable_heat_incentive: RenewableHeatIncentive
     draughtproofed_door_count: int
-    mechanical_vent_duct_type: int
+    mechanical_vent_duct_type: Optional[int] = None
-    windows_transmission_details: WindowsTransmissionDetails
+    windows_transmission_details: Optional[WindowsTransmissionDetails] = None
-    cfl_fixed_lighting_bulbs_count: int
+    cfl_fixed_lighting_bulbs_count: Optional[int] = None
     energy_consumption_current: int
     has_fixed_air_conditioning: str
     multiple_glazed_proportion: int
     calculation_software_version: str
     energy_consumption_potential: int
     environmental_impact_current: int
-    led_fixed_lighting_bulbs_count: int
+    led_fixed_lighting_bulbs_count: Optional[int] = None
-    mechanical_vent_duct_placement: int
+    mechanical_vent_duct_placement: Optional[int] = None
-    mechanical_vent_duct_insulation: int
+    mechanical_vent_duct_insulation: Optional[int] = None
     potential_energy_efficiency_band: str
-    pressure_test_certificate_number: int
+    pressure_test_certificate_number: Optional[int] = None
-    mechanical_ventilation_index_number: int
+    mechanical_ventilation_index_number: Optional[int] = None
     co2_emissions_current_per_floor_area: int
     current_energy_efficiency_band: str
     environmental_impact_potential: int
-    low_energy_fixed_lighting_bulbs_count: int
+    low_energy_fixed_lighting_bulbs_count: Optional[int] = None
-    mechanical_vent_duct_insulation_level: int
+    mechanical_vent_duct_insulation_level: Optional[int] = None
-    mechanical_vent_measured_installation: str
+    mechanical_vent_measured_installation: Optional[str] = None
     incandescent_fixed_lighting_bulbs_count: int
     sap_flat_details: Optional[SapFlatDetails] = None
     addendum: Optional[Addendum] = None