mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-27 23:35:01 +00:00
Merge pull request #1674 from Hestia-Homes/fix/epc-persistence-dropped-fields
EPC persistence blind spot: persist dropped calculator-read fields + round-trip gate (#1665)
This commit is contained in:
KhalimCK
GitHub
commit
44072ed159
4 changed files with 466 additions and 35 deletions
|
|
@ -1,7 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar, Optional, Union
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import BigInteger, Column
|
||||
from sqlalchemy import Enum as SAEnum
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlmodel import SQLModel, Field
|
||||
|
|
@ -15,6 +15,9 @@ from datatypes.epc.domain.epc_property_data import (
|
|||
SapBuildingPart,
|
||||
SapFloorDimension,
|
||||
SapFlatDetails,
|
||||
SapRoofWindow,
|
||||
SapRoomInRoof,
|
||||
SapRoomInRoofSurface,
|
||||
SapWindow,
|
||||
)
|
||||
|
||||
|
|
@ -36,7 +39,12 @@ class EpcPropertyModel(SQLModel, table=True):
|
|||
source: str = Field(default="lodged")
|
||||
|
||||
# Identity / admin
|
||||
uprn: Optional[int] = Field(default=None)
|
||||
# BIGINT: real 12-digit UPRNs overflow int32 (the FE `property.uprn` is
|
||||
# already bigint). SQLModel maps `int` -> INTEGER by default, so pin the
|
||||
# column type or every real-UPRN insert hard-fails NumericValueOutOfRange.
|
||||
uprn: Optional[int] = Field(
|
||||
default=None, sa_column=Column(BigInteger, nullable=True)
|
||||
)
|
||||
uprn_source: Optional[str] = Field(default=None)
|
||||
report_reference: Optional[str] = Field(default=None)
|
||||
report_type: Optional[str] = Field(default=None)
|
||||
|
|
@ -168,6 +176,9 @@ class EpcPropertyModel(SQLModel, table=True):
|
|||
)
|
||||
heating_cylinder_insulation_thickness_mm: Optional[int] = Field(default=None)
|
||||
heating_cylinder_volume_measured_l: Optional[int] = Field(default=None)
|
||||
# #1665: read by cert_to_inputs (§4 cylinder loss); dropping it billed the
|
||||
# cylinder at the Table 2b age-band default (worst case -15.53 SAP).
|
||||
heating_cylinder_heat_loss: Optional[float] = Field(default=None)
|
||||
heating_wwhrs_index_number_1: Optional[int] = Field(default=None)
|
||||
heating_wwhrs_index_number_2: Optional[int] = Field(default=None)
|
||||
heating_shower_outlet_type: Optional[Union[int, str]] = Field(
|
||||
|
|
@ -359,6 +370,7 @@ class EpcPropertyModel(SQLModel, table=True):
|
|||
heating_secondary_heating_type=h.secondary_heating_type,
|
||||
heating_cylinder_insulation_thickness_mm=h.cylinder_insulation_thickness_mm,
|
||||
heating_cylinder_volume_measured_l=h.cylinder_volume_measured_l,
|
||||
heating_cylinder_heat_loss=h.cylinder_heat_loss,
|
||||
heating_wwhrs_index_number_1=h.instantaneous_wwhrs.wwhrs_index_number1,
|
||||
heating_wwhrs_index_number_2=h.instantaneous_wwhrs.wwhrs_index_number2,
|
||||
heating_shower_outlet_type=shower.shower_outlet_type if shower else None,
|
||||
|
|
@ -668,6 +680,11 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
# to the gov-API code-6 heuristic", `False` means "explicitly system-built,
|
||||
# do NOT apply the heuristic". Collapsing False to None inverts the answer.
|
||||
wall_is_basement: Optional[bool] = Field(default=None)
|
||||
# #1665: lodged wall U (gov-EPC `wall_u_value`), authoritative over the
|
||||
# derived default when present; dropped it fell back to the age-band default
|
||||
# (-2.32 SAP). The prior allowlist "RdSAP uses defaults" was stale — the
|
||||
# cascade does honour the lodged U on the API path.
|
||||
wall_u_value: Optional[float] = Field(default=None)
|
||||
room_in_roof_floor_area: Optional[float] = Field(default=None)
|
||||
room_in_roof_construction_age_band: Optional[str] = Field(default=None)
|
||||
alt_wall_1_area: Optional[float] = Field(default=None)
|
||||
|
|
@ -677,6 +694,12 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
alt_wall_1_thickness_measured: Optional[str] = Field(default=None)
|
||||
alt_wall_1_insulation_thickness: Optional[str] = Field(default=None)
|
||||
alt_wall_1_is_sheltered: Optional[bool] = Field(default=None)
|
||||
# #1665: calculator-read alt-wall fields (heat_transmission.py) with no column.
|
||||
alt_wall_1_u_value: Optional[float] = Field(default=None)
|
||||
alt_wall_1_thickness_mm: Optional[int] = Field(default=None)
|
||||
# Nullable is load-bearing: None ("not stated") != False ("not a basement").
|
||||
# A NOT NULL DEFAULT false re-enables the code-6 basement heuristic (cf #1661).
|
||||
alt_wall_1_is_basement: Optional[bool] = Field(default=None)
|
||||
alt_wall_2_area: Optional[float] = Field(default=None)
|
||||
alt_wall_2_dry_lined: Optional[str] = Field(default=None)
|
||||
alt_wall_2_construction: Optional[int] = Field(default=None)
|
||||
|
|
@ -684,6 +707,9 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
alt_wall_2_thickness_measured: Optional[str] = Field(default=None)
|
||||
alt_wall_2_insulation_thickness: Optional[str] = Field(default=None)
|
||||
alt_wall_2_is_sheltered: Optional[bool] = Field(default=None)
|
||||
alt_wall_2_u_value: Optional[float] = Field(default=None)
|
||||
alt_wall_2_thickness_mm: Optional[int] = Field(default=None)
|
||||
alt_wall_2_is_basement: Optional[bool] = Field(default=None)
|
||||
|
||||
@classmethod
|
||||
def from_domain(
|
||||
|
|
@ -719,6 +745,7 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
roof_insulation_thickness=part.roof_insulation_thickness,
|
||||
rafter_insulation_thickness=part.rafter_insulation_thickness,
|
||||
wall_is_basement=part.wall_is_basement,
|
||||
wall_u_value=part.wall_u_value,
|
||||
room_in_roof_floor_area=float(rir.floor_area) if rir else None,
|
||||
room_in_roof_construction_age_band=(
|
||||
rir.construction_age_band if rir else None
|
||||
|
|
@ -732,6 +759,9 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
aw1.wall_insulation_thickness if aw1 else None
|
||||
),
|
||||
alt_wall_1_is_sheltered=aw1.is_sheltered if aw1 else None,
|
||||
alt_wall_1_u_value=aw1.u_value if aw1 else None,
|
||||
alt_wall_1_thickness_mm=aw1.wall_thickness_mm if aw1 else None,
|
||||
alt_wall_1_is_basement=aw1.is_basement if aw1 else None,
|
||||
alt_wall_2_area=aw2.wall_area if aw2 else None,
|
||||
alt_wall_2_dry_lined=aw2.wall_dry_lined if aw2 else None,
|
||||
alt_wall_2_construction=aw2.wall_construction if aw2 else None,
|
||||
|
|
@ -741,6 +771,9 @@ class EpcBuildingPartModel(SQLModel, table=True):
|
|||
aw2.wall_insulation_thickness if aw2 else None
|
||||
),
|
||||
alt_wall_2_is_sheltered=aw2.is_sheltered if aw2 else None,
|
||||
alt_wall_2_u_value=aw2.u_value if aw2 else None,
|
||||
alt_wall_2_thickness_mm=aw2.wall_thickness_mm if aw2 else None,
|
||||
alt_wall_2_is_basement=aw2.is_basement if aw2 else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -872,6 +905,130 @@ class EpcPhotovoltaicArrayModel(SQLModel, table=True):
|
|||
)
|
||||
|
||||
|
||||
class EpcRoofWindowModel(SQLModel, table=True):
|
||||
__tablename__: ClassVar[str] = (
|
||||
"epc_roof_window" # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
)
|
||||
|
||||
id: Optional[int] = Field(default=None, primary_key=True)
|
||||
epc_property_id: int = Field(foreign_key="epc_property.id", nullable=False)
|
||||
|
||||
# List position on `sap_roof_windows`, persisted so the order is recoverable
|
||||
# for round-trip equality (#1665). Fields mirror SapRoofWindow: area/U feed
|
||||
# §3 (27a) heat transmission; orientation/pitch/g/frame_factor feed §6 solar.
|
||||
roof_window_index: int
|
||||
area_m2: float
|
||||
u_value_raw: float
|
||||
orientation: int
|
||||
pitch_deg: float
|
||||
g_perpendicular: float
|
||||
frame_factor: float
|
||||
glazing_type: int
|
||||
# SAP building-part index — int on the API path, str on site-notes; JSONB
|
||||
# preserves the int-vs-str distinction (mirrors SapWindow.window_location).
|
||||
window_location: Union[int, str] = Field(
|
||||
sa_column=Column(JSONB, nullable=False)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_domain(
|
||||
cls, rw: SapRoofWindow, roof_window_index: int, epc_property_id: int
|
||||
) -> EpcRoofWindowModel:
|
||||
return cls(
|
||||
epc_property_id=epc_property_id,
|
||||
roof_window_index=roof_window_index,
|
||||
area_m2=rw.area_m2,
|
||||
u_value_raw=rw.u_value_raw,
|
||||
orientation=rw.orientation,
|
||||
pitch_deg=rw.pitch_deg,
|
||||
g_perpendicular=rw.g_perpendicular,
|
||||
frame_factor=rw.frame_factor,
|
||||
glazing_type=rw.glazing_type,
|
||||
window_location=rw.window_location,
|
||||
)
|
||||
|
||||
|
||||
class EpcRoomInRoofModel(SQLModel, table=True):
|
||||
__tablename__: ClassVar[str] = (
|
||||
"epc_room_in_roof" # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
)
|
||||
|
||||
id: Optional[int] = Field(default=None, primary_key=True)
|
||||
# 0..1 per building part (SapBuildingPart.sap_room_in_roof is Optional, not a
|
||||
# list) — enforced UNIQUE by the FE migration. #1665 + #1664.
|
||||
epc_building_part_id: int = Field(
|
||||
foreign_key="epc_building_part.id", nullable=False
|
||||
)
|
||||
# floor_area is Union[int, float] in the domain; int-vs-float is not
|
||||
# semantically load-bearing (the calculator does float(rir.floor_area)).
|
||||
floor_area: float
|
||||
construction_age_band: str
|
||||
# §3.9.2 Simplified Type 2 gable/common-wall scalars (dormant on the API
|
||||
# path — 0 corpus — but populated by the Elmhurst/site-notes path).
|
||||
common_wall_length_m: Optional[float] = Field(default=None)
|
||||
common_wall_height_m: Optional[float] = Field(default=None)
|
||||
gable_1_length_m: Optional[float] = Field(default=None)
|
||||
gable_1_height_m: Optional[float] = Field(default=None)
|
||||
gable_2_length_m: Optional[float] = Field(default=None)
|
||||
gable_2_height_m: Optional[float] = Field(default=None)
|
||||
|
||||
@classmethod
|
||||
def from_domain(
|
||||
cls, rir: SapRoomInRoof, epc_building_part_id: int
|
||||
) -> EpcRoomInRoofModel:
|
||||
return cls(
|
||||
epc_building_part_id=epc_building_part_id,
|
||||
floor_area=float(rir.floor_area),
|
||||
construction_age_band=rir.construction_age_band,
|
||||
common_wall_length_m=rir.common_wall_length_m,
|
||||
common_wall_height_m=rir.common_wall_height_m,
|
||||
gable_1_length_m=rir.gable_1_length_m,
|
||||
gable_1_height_m=rir.gable_1_height_m,
|
||||
gable_2_length_m=rir.gable_2_length_m,
|
||||
gable_2_height_m=rir.gable_2_height_m,
|
||||
)
|
||||
|
||||
|
||||
class EpcRoomInRoofSurfaceModel(SQLModel, table=True):
|
||||
__tablename__: ClassVar[str] = (
|
||||
"epc_room_in_roof_surface" # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
)
|
||||
|
||||
id: Optional[int] = Field(default=None, primary_key=True)
|
||||
epc_room_in_roof_id: int = Field(
|
||||
foreign_key="epc_room_in_roof.id", nullable=False
|
||||
)
|
||||
# List position on `detailed_surfaces`, persisted for round-trip order.
|
||||
surface_index: int
|
||||
kind: str
|
||||
# Can be NEGATIVE: a §3.9.2 absent-gable adjustment produces a signed area
|
||||
# that deducts from the A_RR residual — no CHECK >= 0.
|
||||
area_m2: float
|
||||
# Null vs 0 is the U-value branch (Table 17 measured row vs Table 18 col (4)
|
||||
# age-band default) — keep nullable, never DEFAULT 0.
|
||||
insulation_thickness_mm: Optional[int] = Field(default=None)
|
||||
insulation_type: Optional[str] = Field(default=None)
|
||||
# Assessor U override; null falls through to the Table 4 cascade. Null != 0.
|
||||
u_value: Optional[float] = Field(default=None)
|
||||
|
||||
@classmethod
|
||||
def from_domain(
|
||||
cls,
|
||||
surface: SapRoomInRoofSurface,
|
||||
surface_index: int,
|
||||
epc_room_in_roof_id: int,
|
||||
) -> EpcRoomInRoofSurfaceModel:
|
||||
return cls(
|
||||
epc_room_in_roof_id=epc_room_in_roof_id,
|
||||
surface_index=surface_index,
|
||||
kind=surface.kind,
|
||||
area_m2=surface.area_m2,
|
||||
insulation_thickness_mm=surface.insulation_thickness_mm,
|
||||
insulation_type=surface.insulation_type,
|
||||
u_value=surface.u_value,
|
||||
)
|
||||
|
||||
|
||||
class EpcEnergyElementModel(SQLModel, table=True):
|
||||
__tablename__: ClassVar[str] = (
|
||||
"epc_energy_element" # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ from datatypes.epc.domain.epc_property_data import (
|
|||
SapFlatDetails,
|
||||
SapFloorDimension,
|
||||
SapHeating,
|
||||
SapRoofWindow,
|
||||
SapRoomInRoof,
|
||||
SapRoomInRoofSurface,
|
||||
SapVentilation,
|
||||
SapWindow,
|
||||
ShowerOutlet,
|
||||
|
|
@ -48,6 +50,9 @@ from infrastructure.postgres.epc_property_table import (
|
|||
EpcPropertyEnergyPerformanceModel,
|
||||
EpcPropertyModel,
|
||||
EpcRenewableHeatIncentiveModel,
|
||||
EpcRoofWindowModel,
|
||||
EpcRoomInRoofModel,
|
||||
EpcRoomInRoofSurfaceModel,
|
||||
EpcWindowModel,
|
||||
)
|
||||
from repositories.epc.epc_repository import (
|
||||
|
|
@ -195,6 +200,7 @@ class EpcPostgresRepository(EpcRepository):
|
|||
parts_ordered: list[tuple[Any, int]] = [] # (SapBuildingPart, epc_property_id)
|
||||
window_rows: list[dict[str, Any]] = []
|
||||
pv_rows: list[dict[str, Any]] = []
|
||||
roof_window_rows: list[dict[str, Any]] = []
|
||||
element_rows: list[dict[str, Any]] = []
|
||||
flat_rows: list[dict[str, Any]] = []
|
||||
rhi_rows: list[dict[str, Any]] = []
|
||||
|
|
@ -231,6 +237,13 @@ class EpcPostgresRepository(EpcRepository):
|
|||
frozenset({"id"}),
|
||||
)
|
||||
)
|
||||
for idx, roof_window in enumerate(d.sap_roof_windows or []):
|
||||
roof_window_rows.append(
|
||||
_col_values(
|
||||
EpcRoofWindowModel.from_domain(roof_window, idx, epc_pid),
|
||||
frozenset({"id"}),
|
||||
)
|
||||
)
|
||||
for etype, els in (
|
||||
("roof", d.roofs),
|
||||
("wall", d.walls),
|
||||
|
|
@ -284,6 +297,8 @@ class EpcPostgresRepository(EpcRepository):
|
|||
self._session.execute(_sa_insert(EpcWindowModel), window_rows) # type: ignore[deprecated]
|
||||
if pv_rows:
|
||||
self._session.execute(_sa_insert(EpcPhotovoltaicArrayModel), pv_rows) # type: ignore[deprecated]
|
||||
if roof_window_rows:
|
||||
self._session.execute(_sa_insert(EpcRoofWindowModel), roof_window_rows) # type: ignore[deprecated]
|
||||
if element_rows:
|
||||
self._session.execute(_sa_insert(EpcEnergyElementModel), element_rows) # type: ignore[deprecated]
|
||||
if flat_rows:
|
||||
|
|
@ -318,6 +333,36 @@ class EpcPostgresRepository(EpcRepository):
|
|||
if floor_rows:
|
||||
self._session.execute(_sa_insert(EpcFloorDimensionModel), floor_rows) # type: ignore[deprecated]
|
||||
|
||||
# Room-in-roof: 0..1 per building part, each with 0..n surfaces.
|
||||
# Insert the RIR rows with RETURNING, then zip to resolve the
|
||||
# surface FKs (same positional-zip contract as the parts above).
|
||||
rir_pairs = [
|
||||
(part.sap_room_in_roof, bp_row[0])
|
||||
for (part, _), bp_row in zip(parts_ordered, returned_bps)
|
||||
if part.sap_room_in_roof is not None
|
||||
]
|
||||
if rir_pairs:
|
||||
rir_rows = [
|
||||
_col_values(
|
||||
EpcRoomInRoofModel.from_domain(rir, bp_id), frozenset({"id"})
|
||||
)
|
||||
for rir, bp_id in rir_pairs
|
||||
]
|
||||
returned_rirs = self._session.execute( # type: ignore[deprecated]
|
||||
_sa_insert(EpcRoomInRoofModel).returning(EpcRoomInRoofModel.__table__.c["id"]), # type: ignore[attr-defined]
|
||||
rir_rows,
|
||||
).all()
|
||||
surface_rows = [
|
||||
_col_values(
|
||||
EpcRoomInRoofSurfaceModel.from_domain(surface, s_idx, rir_row[0]),
|
||||
frozenset({"id"}),
|
||||
)
|
||||
for (rir, _), rir_row in zip(rir_pairs, returned_rirs)
|
||||
for s_idx, surface in enumerate(rir.detailed_surfaces or [])
|
||||
]
|
||||
if surface_rows:
|
||||
self._session.execute(_sa_insert(EpcRoomInRoofSurfaceModel), surface_rows) # type: ignore[deprecated]
|
||||
|
||||
return epc_property_ids
|
||||
|
||||
def _delete_for_properties(
|
||||
|
|
@ -347,6 +392,29 @@ class EpcPostgresRepository(EpcRepository):
|
|||
if i is not None
|
||||
]
|
||||
if part_ids:
|
||||
# Room-in-roof surfaces FK the RIR row, which FKs the building part;
|
||||
# with ON DELETE no action we must clear the graph bottom-up
|
||||
# (surfaces -> room_in_roof) before the building parts below.
|
||||
rir_ids = [
|
||||
i
|
||||
for i in self._session.exec(
|
||||
select(EpcRoomInRoofModel.id).where(
|
||||
col(EpcRoomInRoofModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
).all()
|
||||
if i is not None
|
||||
]
|
||||
if rir_ids:
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcRoomInRoofSurfaceModel).where(
|
||||
col(EpcRoomInRoofSurfaceModel.epc_room_in_roof_id).in_(rir_ids)
|
||||
)
|
||||
)
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcRoomInRoofModel).where(
|
||||
col(EpcRoomInRoofModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
)
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcFloorDimensionModel).where(
|
||||
col(EpcFloorDimensionModel.epc_building_part_id).in_(part_ids)
|
||||
|
|
@ -359,6 +427,7 @@ class EpcPostgresRepository(EpcRepository):
|
|||
EpcBuildingPartModel,
|
||||
EpcWindowModel,
|
||||
EpcPhotovoltaicArrayModel,
|
||||
EpcRoofWindowModel,
|
||||
EpcFlatDetailsModel,
|
||||
EpcRenewableHeatIncentiveModel,
|
||||
):
|
||||
|
|
@ -394,6 +463,29 @@ class EpcPostgresRepository(EpcRepository):
|
|||
if i is not None
|
||||
]
|
||||
if part_ids:
|
||||
# Room-in-roof surfaces FK the RIR row, which FKs the building part;
|
||||
# with ON DELETE no action we must clear the graph bottom-up
|
||||
# (surfaces -> room_in_roof) before the building parts below.
|
||||
rir_ids = [
|
||||
i
|
||||
for i in self._session.exec(
|
||||
select(EpcRoomInRoofModel.id).where(
|
||||
col(EpcRoomInRoofModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
).all()
|
||||
if i is not None
|
||||
]
|
||||
if rir_ids:
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcRoomInRoofSurfaceModel).where(
|
||||
col(EpcRoomInRoofSurfaceModel.epc_room_in_roof_id).in_(rir_ids)
|
||||
)
|
||||
)
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcRoomInRoofModel).where(
|
||||
col(EpcRoomInRoofModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
)
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcFloorDimensionModel).where(
|
||||
col(EpcFloorDimensionModel.epc_building_part_id).in_(part_ids)
|
||||
|
|
@ -406,6 +498,7 @@ class EpcPostgresRepository(EpcRepository):
|
|||
EpcBuildingPartModel,
|
||||
EpcWindowModel,
|
||||
EpcPhotovoltaicArrayModel,
|
||||
EpcRoofWindowModel,
|
||||
EpcFlatDetailsModel,
|
||||
EpcRenewableHeatIncentiveModel,
|
||||
):
|
||||
|
|
@ -553,10 +646,21 @@ class EpcPostgresRepository(EpcRepository):
|
|||
.order_by(EpcPhotovoltaicArrayModel.array_index) # type: ignore[arg-type]
|
||||
).all()
|
||||
)
|
||||
roof_windows_by = _group_by_epc(
|
||||
self._session.exec(
|
||||
select(EpcRoofWindowModel)
|
||||
.where(col(EpcRoofWindowModel.epc_property_id).in_(epc_ids))
|
||||
.order_by(EpcRoofWindowModel.roof_window_index) # type: ignore[arg-type]
|
||||
).all()
|
||||
)
|
||||
part_ids = [
|
||||
bp.id for parts in parts_by.values() for bp in parts if bp.id is not None
|
||||
]
|
||||
floor_dims_by_part = self._floor_dims_by_part(part_ids)
|
||||
rir_by_part = self._rir_by_part(part_ids)
|
||||
surfaces_by_rir = self._rir_surfaces_by_rir(
|
||||
[r.id for r in rir_by_part.values() if r.id is not None]
|
||||
)
|
||||
|
||||
result: dict[int, EpcPropertyData] = {}
|
||||
for key, parent in parents_by_key.items():
|
||||
|
|
@ -568,8 +672,11 @@ class EpcPostgresRepository(EpcRepository):
|
|||
heating_rows=heating_by.get(epc_id, []),
|
||||
part_rows=parts_by.get(epc_id, []),
|
||||
floor_dims_by_part=floor_dims_by_part,
|
||||
rir_by_part=rir_by_part,
|
||||
surfaces_by_rir=surfaces_by_rir,
|
||||
window_rows=windows_by.get(epc_id, []),
|
||||
pv_array_rows=pv_arrays_by.get(epc_id, []),
|
||||
roof_window_rows=roof_windows_by.get(epc_id, []),
|
||||
flat_row=flat_by.get(epc_id),
|
||||
rhi_row=rhi_by.get(epc_id),
|
||||
)
|
||||
|
|
@ -590,6 +697,33 @@ class EpcPostgresRepository(EpcRepository):
|
|||
grouped.setdefault(row.epc_building_part_id, []).append(row)
|
||||
return grouped
|
||||
|
||||
@private
|
||||
def _rir_by_part(self, part_ids: list[int]) -> dict[int, EpcRoomInRoofModel]:
|
||||
if not part_ids:
|
||||
return {}
|
||||
rows = self._session.exec(
|
||||
select(EpcRoomInRoofModel).where(
|
||||
col(EpcRoomInRoofModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
).all()
|
||||
return {row.epc_building_part_id: row for row in rows}
|
||||
|
||||
@private
|
||||
def _rir_surfaces_by_rir(
|
||||
self, rir_ids: list[int]
|
||||
) -> dict[int, list[EpcRoomInRoofSurfaceModel]]:
|
||||
if not rir_ids:
|
||||
return {}
|
||||
rows = self._session.exec(
|
||||
select(EpcRoomInRoofSurfaceModel)
|
||||
.where(col(EpcRoomInRoofSurfaceModel.epc_room_in_roof_id).in_(rir_ids))
|
||||
.order_by(EpcRoomInRoofSurfaceModel.surface_index) # type: ignore[arg-type]
|
||||
).all()
|
||||
grouped: dict[int, list[EpcRoomInRoofSurfaceModel]] = {}
|
||||
for row in rows:
|
||||
grouped.setdefault(row.epc_room_in_roof_id, []).append(row)
|
||||
return grouped
|
||||
|
||||
def get(self, epc_property_id: int) -> EpcPropertyData:
|
||||
p = self._session.get(EpcPropertyModel, epc_property_id)
|
||||
if p is None:
|
||||
|
|
@ -632,8 +766,12 @@ class EpcPostgresRepository(EpcRepository):
|
|||
).first()
|
||||
window_rows = self._windows(epc_property_id)
|
||||
pv_array_rows = self._pv_arrays(epc_property_id)
|
||||
floor_dims_by_part = self._floor_dims_by_part(
|
||||
[bp.id for bp in part_rows if bp.id is not None]
|
||||
roof_window_rows = self._roof_windows(epc_property_id)
|
||||
part_ids = [bp.id for bp in part_rows if bp.id is not None]
|
||||
floor_dims_by_part = self._floor_dims_by_part(part_ids)
|
||||
rir_by_part = self._rir_by_part(part_ids)
|
||||
surfaces_by_rir = self._rir_surfaces_by_rir(
|
||||
[r.id for r in rir_by_part.values() if r.id is not None]
|
||||
)
|
||||
return self._compose(
|
||||
p=p,
|
||||
|
|
@ -642,8 +780,11 @@ class EpcPostgresRepository(EpcRepository):
|
|||
heating_rows=heating_rows,
|
||||
part_rows=part_rows,
|
||||
floor_dims_by_part=floor_dims_by_part,
|
||||
rir_by_part=rir_by_part,
|
||||
surfaces_by_rir=surfaces_by_rir,
|
||||
window_rows=window_rows,
|
||||
pv_array_rows=pv_array_rows,
|
||||
roof_window_rows=roof_window_rows,
|
||||
flat_row=flat_row,
|
||||
rhi_row=rhi_row,
|
||||
)
|
||||
|
|
@ -657,8 +798,11 @@ class EpcPostgresRepository(EpcRepository):
|
|||
heating_rows: list[EpcMainHeatingDetailModel],
|
||||
part_rows: list[EpcBuildingPartModel],
|
||||
floor_dims_by_part: dict[int, list[EpcFloorDimensionModel]],
|
||||
rir_by_part: dict[int, EpcRoomInRoofModel],
|
||||
surfaces_by_rir: dict[int, list[EpcRoomInRoofSurfaceModel]],
|
||||
window_rows: list[EpcWindowModel],
|
||||
pv_array_rows: list[EpcPhotovoltaicArrayModel],
|
||||
roof_window_rows: list[EpcRoofWindowModel],
|
||||
flat_row: Optional[EpcFlatDetailsModel],
|
||||
rhi_row: Optional[EpcRenewableHeatIncentiveModel],
|
||||
) -> EpcPropertyData:
|
||||
|
|
@ -688,10 +832,18 @@ class EpcPostgresRepository(EpcRepository):
|
|||
door_count=p.door_count,
|
||||
sap_heating=self._to_sap_heating(p, heating_rows),
|
||||
sap_windows=[self._to_window(w) for w in window_rows],
|
||||
sap_roof_windows=(
|
||||
[self._to_roof_window(r) for r in roof_window_rows]
|
||||
if roof_window_rows
|
||||
else None
|
||||
),
|
||||
sap_energy_source=self._to_energy_source(p, pv_array_rows),
|
||||
sap_building_parts=[
|
||||
self._to_building_part(
|
||||
bp, floor_dims_by_part.get(bp.id, []) if bp.id is not None else []
|
||||
bp,
|
||||
floor_dims_by_part.get(bp.id, []) if bp.id is not None else [],
|
||||
rir_by_part.get(bp.id) if bp.id is not None else None,
|
||||
surfaces_by_rir,
|
||||
)
|
||||
for bp in part_rows
|
||||
],
|
||||
|
|
@ -869,6 +1021,29 @@ class EpcPostgresRepository(EpcRepository):
|
|||
).all()
|
||||
)
|
||||
|
||||
@private
|
||||
def _roof_windows(self, epc_property_id: int) -> list[EpcRoofWindowModel]:
|
||||
return list(
|
||||
self._session.exec(
|
||||
select(EpcRoofWindowModel)
|
||||
.where(EpcRoofWindowModel.epc_property_id == epc_property_id)
|
||||
.order_by(EpcRoofWindowModel.roof_window_index) # type: ignore[arg-type]
|
||||
).all()
|
||||
)
|
||||
|
||||
@private
|
||||
def _to_roof_window(self, r: EpcRoofWindowModel) -> SapRoofWindow:
|
||||
return SapRoofWindow(
|
||||
area_m2=r.area_m2,
|
||||
u_value_raw=r.u_value_raw,
|
||||
orientation=r.orientation,
|
||||
pitch_deg=r.pitch_deg,
|
||||
g_perpendicular=r.g_perpendicular,
|
||||
frame_factor=r.frame_factor,
|
||||
glazing_type=r.glazing_type,
|
||||
window_location=r.window_location,
|
||||
)
|
||||
|
||||
@private
|
||||
def _to_energy_element(self, e: EpcEnergyElementModel) -> EnergyElement:
|
||||
return EnergyElement(
|
||||
|
|
@ -909,6 +1084,7 @@ class EpcPostgresRepository(EpcRepository):
|
|||
secondary_heating_type=p.heating_secondary_heating_type,
|
||||
cylinder_insulation_thickness_mm=p.heating_cylinder_insulation_thickness_mm,
|
||||
cylinder_volume_measured_l=p.heating_cylinder_volume_measured_l,
|
||||
cylinder_heat_loss=p.heating_cylinder_heat_loss,
|
||||
number_baths=p.heating_number_baths,
|
||||
number_baths_wwhrs=p.heating_number_baths_wwhrs,
|
||||
electric_shower_count=p.heating_electric_shower_count,
|
||||
|
|
@ -974,7 +1150,11 @@ class EpcPostgresRepository(EpcRepository):
|
|||
|
||||
@private
|
||||
def _to_building_part(
|
||||
self, bp: EpcBuildingPartModel, floor_rows: list[EpcFloorDimensionModel]
|
||||
self,
|
||||
bp: EpcBuildingPartModel,
|
||||
floor_rows: list[EpcFloorDimensionModel],
|
||||
rir_model: Optional[EpcRoomInRoofModel],
|
||||
surfaces_by_rir: dict[int, list[EpcRoomInRoofSurfaceModel]],
|
||||
) -> SapBuildingPart:
|
||||
return SapBuildingPart(
|
||||
identifier=BuildingPartIdentifier(bp.identifier),
|
||||
|
|
@ -1005,19 +1185,50 @@ class EpcPostgresRepository(EpcRepository):
|
|||
roof_insulation_thickness=bp.roof_insulation_thickness,
|
||||
rafter_insulation_thickness=bp.rafter_insulation_thickness,
|
||||
wall_is_basement=bp.wall_is_basement,
|
||||
sap_room_in_roof=(
|
||||
SapRoomInRoof(
|
||||
floor_area=bp.room_in_roof_floor_area,
|
||||
construction_age_band=_require(
|
||||
bp.room_in_roof_construction_age_band,
|
||||
"room_in_roof_construction_age_band",
|
||||
),
|
||||
)
|
||||
if bp.room_in_roof_floor_area is not None
|
||||
else None
|
||||
wall_u_value=bp.wall_u_value,
|
||||
# Reconstruct from the epc_room_in_roof child table (full geometry
|
||||
# incl. detailed_surfaces); the flat room_in_roof_* columns on
|
||||
# epc_building_part are superseded and dropped in a follow-up (#1664).
|
||||
sap_room_in_roof=self._to_room_in_roof(rir_model, surfaces_by_rir),
|
||||
)
|
||||
|
||||
@private
|
||||
def _to_room_in_roof(
|
||||
self,
|
||||
rir_model: Optional[EpcRoomInRoofModel],
|
||||
surfaces_by_rir: dict[int, list[EpcRoomInRoofSurfaceModel]],
|
||||
) -> Optional[SapRoomInRoof]:
|
||||
if rir_model is None:
|
||||
return None
|
||||
surfaces = (
|
||||
surfaces_by_rir.get(rir_model.id, []) if rir_model.id is not None else []
|
||||
)
|
||||
return SapRoomInRoof(
|
||||
floor_area=rir_model.floor_area,
|
||||
construction_age_band=rir_model.construction_age_band,
|
||||
common_wall_length_m=rir_model.common_wall_length_m,
|
||||
common_wall_height_m=rir_model.common_wall_height_m,
|
||||
gable_1_length_m=rir_model.gable_1_length_m,
|
||||
gable_1_height_m=rir_model.gable_1_height_m,
|
||||
gable_2_length_m=rir_model.gable_2_length_m,
|
||||
gable_2_height_m=rir_model.gable_2_height_m,
|
||||
detailed_surfaces=(
|
||||
[self._to_rir_surface(s) for s in surfaces] if surfaces else None
|
||||
),
|
||||
)
|
||||
|
||||
@private
|
||||
def _to_rir_surface(
|
||||
self, s: EpcRoomInRoofSurfaceModel
|
||||
) -> SapRoomInRoofSurface:
|
||||
return SapRoomInRoofSurface(
|
||||
kind=s.kind,
|
||||
area_m2=s.area_m2,
|
||||
insulation_thickness_mm=s.insulation_thickness_mm,
|
||||
insulation_type=s.insulation_type,
|
||||
u_value=s.u_value,
|
||||
)
|
||||
|
||||
@private
|
||||
def _to_alt_wall(
|
||||
self, bp: EpcBuildingPartModel, n: int
|
||||
|
|
@ -1043,6 +1254,13 @@ class EpcPostgresRepository(EpcRepository):
|
|||
else bp.alt_wall_2_insulation_thickness
|
||||
)
|
||||
sheltered = bp.alt_wall_1_is_sheltered if n == 1 else bp.alt_wall_2_is_sheltered
|
||||
u_value = bp.alt_wall_1_u_value if n == 1 else bp.alt_wall_2_u_value
|
||||
thickness_mm = (
|
||||
bp.alt_wall_1_thickness_mm if n == 1 else bp.alt_wall_2_thickness_mm
|
||||
)
|
||||
is_basement = (
|
||||
bp.alt_wall_1_is_basement if n == 1 else bp.alt_wall_2_is_basement
|
||||
)
|
||||
return SapAlternativeWall(
|
||||
wall_area=area,
|
||||
wall_dry_lined=_require(dry_lined, f"alt_wall_{n}_dry_lined"),
|
||||
|
|
@ -1057,6 +1275,9 @@ class EpcPostgresRepository(EpcRepository):
|
|||
# Nullable column (added later than the alt wall itself); a row from
|
||||
# before the column existed reads None → the domain default False.
|
||||
is_sheltered=sheltered if sheltered is not None else False,
|
||||
u_value=u_value,
|
||||
wall_thickness_mm=thickness_mm,
|
||||
is_basement=is_basement,
|
||||
)
|
||||
|
||||
@private
|
||||
|
|
|
|||
|
|
@ -40,27 +40,9 @@ _UNPERSISTED_ALLOWLIST: dict[str, str] = {
|
|||
"EpcPropertyData.lzc_energy_sources": "dormant — not read by the calculator; no FE column",
|
||||
# Scoring-relevant round-trip gaps awaiting FE columns / child tables
|
||||
# (tracked follow-ups; see docs/migrations/epc-property-round-trip-fidelity.md).
|
||||
"EpcPropertyData.sap_roof_windows": "FE child table pending — tracked round-trip gap",
|
||||
"SapRoofWindow": "FE child table pending — tracked round-trip gap (sap_roof_windows)",
|
||||
"EpcPropertyData.solar_hw_collector_orientation": "FE column pending — tracked round-trip gap",
|
||||
"EpcPropertyData.solar_hw_collector_pitch_deg": "FE column pending — tracked round-trip gap",
|
||||
"EpcPropertyData.solar_hw_overshading": "FE column pending — tracked round-trip gap",
|
||||
"SapAlternativeWall.u_value": "FE column pending — tracked round-trip gap",
|
||||
"SapAlternativeWall.wall_thickness_mm": "FE column pending — tracked round-trip gap",
|
||||
# Room-in-roof detail: only floor_area + age_band reconstruct from the
|
||||
# inlined columns; the geometry / surface detail awaits an FE child table
|
||||
# (tracked round-trip gap, docs/migrations §2).
|
||||
"SapRoomInRoof.common_wall_height_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.common_wall_length_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.detailed_surfaces": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.gable_1_height_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.gable_1_length_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.gable_2_height_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoof.gable_2_length_m": "FE child table pending — room-in-roof detail",
|
||||
"SapRoomInRoofSurface": "FE child table pending — room-in-roof surface detail",
|
||||
# --- Nested gaps the calculator does NOT read (dormant); no FE column.
|
||||
"SapAlternativeWall.is_basement": "dormant — not read by the calculator; no FE column",
|
||||
"SapHeating.cylinder_heat_loss": "dormant — not read by the calculator; no FE column",
|
||||
# --- DELIBERATELY not persisted (#1656). These four ARE read by the
|
||||
# calculator (`heat_transmission.py:994` wall U, `:1068` roof U, `:1118`
|
||||
# floor U, `:955` basement wall) — do NOT re-label them "not read by the
|
||||
|
|
@ -77,7 +59,6 @@ _UNPERSISTED_ALLOWLIST: dict[str, str] = {
|
|||
# `wall_is_basement` is deliberately NOT in this list — it is a
|
||||
# disambiguation flag selecting which RdSAP default applies (§5.17/Table 23),
|
||||
# not a measured U-value, and it IS persisted.
|
||||
"SapBuildingPart.wall_u_value": "deliberate — full-SAP only; RdSAP re-model uses default U-values (#1656)",
|
||||
"SapBuildingPart.roof_u_value": "deliberate — full-SAP only; RdSAP re-model uses default U-values (#1656)",
|
||||
"SapBuildingPart.floor_u_value": "deliberate — full-SAP only; RdSAP re-model uses default U-values (#1656)",
|
||||
}
|
||||
|
|
|
|||
72
tests/repositories/epc/test_epc_roundtrip_corpus_gate.py
Normal file
72
tests/repositories/epc/test_epc_roundtrip_corpus_gate.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""Durable DB-round-trip gate for the persistence blind spot (#1665).
|
||||
|
||||
Production scores a dwelling AFTER a Postgres save+reload; the accuracy corpus
|
||||
(`test_sap_accuracy_corpus.py`) scores straight off the mapped fixture and never
|
||||
saves, so it is structurally blind to any field the calculator reads but the
|
||||
schema drops. This gate closes that hole: every corpus cert is mapped, saved to
|
||||
the ephemeral test DB, read back, and re-scored — and the reloaded SAP must equal
|
||||
the in-memory SAP to 1e-9. Any field the calculator reads but the schema drops
|
||||
makes the two diverge, so a new persistence blind spot fails here immediately
|
||||
instead of being found by hand months later (it would have caught
|
||||
cylinder_heat_loss / room-in-roof / roof-windows / wall_u_value on day one).
|
||||
|
||||
Save+read+rollback per cert keeps the graph off disk (bounded, ~18s / 1000 certs).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Engine
|
||||
from sqlmodel import Session
|
||||
|
||||
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
|
||||
from domain.sap10_calculator.calculator import Sap10Calculator
|
||||
from repositories.epc.epc_postgres_repository import EpcPostgresRepository
|
||||
|
||||
_CORPUS = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "backend/epc_api/json_samples/RdSAP-Schema-21.0.1/corpus.jsonl"
|
||||
)
|
||||
_TOLERANCE = 1e-9
|
||||
|
||||
|
||||
def test_round_trip_preserves_sap_over_the_corpus(db_engine: Engine) -> None:
|
||||
# Arrange
|
||||
calc = Sap10Calculator()
|
||||
lines = _CORPUS.read_text().splitlines()
|
||||
|
||||
# Act — map -> save -> reload -> re-score every cert, collecting any drift.
|
||||
diverged: list[str] = []
|
||||
for line in lines:
|
||||
raw: dict[str, Any] = json.loads(line)
|
||||
epc = EpcPropertyDataMapper.from_api_response(raw)
|
||||
if epc is None:
|
||||
continue
|
||||
try:
|
||||
direct = calc.calculate(epc).sap_score_continuous
|
||||
except Exception:
|
||||
continue
|
||||
# Save -> flush (assigns FKs, writes the graph within the txn) -> read
|
||||
# back in the SAME session -> roll back, so the round-trip exercises the
|
||||
# real save/reconstruct without accumulating 1000 graphs on disk.
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
epc_property_id = repo.save(epc)
|
||||
session.flush()
|
||||
reloaded = repo.get(epc_property_id)
|
||||
session.rollback()
|
||||
reloaded_sap = calc.calculate(reloaded).sap_score_continuous
|
||||
if abs(reloaded_sap - direct) > _TOLERANCE:
|
||||
diverged.append(
|
||||
f"uprn {raw.get('uprn')}: in-memory {direct:.6f} != reloaded "
|
||||
f"{reloaded_sap:.6f} (Δ {reloaded_sap - direct:+.4f})"
|
||||
)
|
||||
|
||||
# Assert — no field the calculator reads is dropped on the DB round-trip.
|
||||
assert not diverged, (
|
||||
f"{len(diverged)} cert(s) lost calculator-read fields on the DB round-trip:\n"
|
||||
+ "\n".join(diverged[:20])
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue