mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
Merge branch 'main' into audit/schema-mapping-gaps
This commit is contained in:
Daniel Roth
commit
d623a29c2d
18 changed files with 1027 additions and 195 deletions
|
|
@ -114,9 +114,27 @@ durable, compounding output of every audit.
|
|||
## Notes
|
||||
|
||||
- Read-only on the DB. `run_modelling_e2e` is a dry run.
|
||||
- **Audit the default plan only** — every check, and every characterisation
|
||||
query, must filter `plan.is_default = TRUE` (the FE-shown plan). The runner's
|
||||
`_QUERY`/`_ROLLUP_QUERY` already do; keep ad-hoc SQL consistent or the counts
|
||||
mix superseded plans into the picture.
|
||||
- **The stored default plan can be STALE vs the live model** — the persisted plan
|
||||
is the output of an *earlier* modelling run; `run_modelling_e2e` re-models
|
||||
against current logic + live EPC/solar. A stored-vs-live gap is the single
|
||||
biggest driver of MEDIUM/HIGH anomalies on a not-yet-re-modelled portfolio, and
|
||||
it spans more checks than the one below names. The fix is operational
|
||||
(**re-model the portfolio, then re-audit**), not a code change — confirm a
|
||||
sample with `run_modelling_e2e` before debugging the calculator.
|
||||
- **Expected, not bugs** (until the override-aware-rebaseline + persistence-fidelity
|
||||
PR deploys and the portfolio is re-modelled): much of `zero-works-post-differs`
|
||||
and `plan-score-below-baseline` is the pre-fix baseline-vs-plan divergence and
|
||||
should shrink after re-model — note it, don't re-debug it.
|
||||
work — see `docs/adr/` — is re-modelled into the portfolio): much of
|
||||
`zero-works-post-differs`, `plan-score-below-baseline`, `already-meets-goal-with-works`,
|
||||
and the non-fuel-switch slice of `negative-bill-savings` is the same stale
|
||||
stored-plan-vs-live divergence and should shrink after re-model — note it,
|
||||
don't re-debug it. Confirm a sample's stored plan differs from `run_modelling_e2e`
|
||||
(stored `air_source_heat_pump` for tens of £k where the live plan is a cheap
|
||||
profitable `solar_pv`; stored works on a property the live model leaves at £0
|
||||
because it already meets goal). The `negative-bill` certs that DO carry an
|
||||
`air_source_heat_pump` are a genuine gas→electric fuel-switch (bills rise at
|
||||
current price ratios) — expected by design.
|
||||
- Adding a check is one decorated `(PropertyAudit) -> Optional[str]` function in
|
||||
`scripts/audit/anomalies.py`; see its module docstring.
|
||||
|
|
|
|||
|
|
@ -29,3 +29,8 @@ moto[s3,sqs]==5.0.28 # mock_aws (moto 5.x) for S3/SQS in orchestration tests
|
|||
black==26.1.0
|
||||
boto3-stubs
|
||||
openai
|
||||
# Type checking — strict pyright gate (CLAUDE.md). The pip `pyright` wrapper uses
|
||||
# the container's Node. pandas-stubs lets pandas-typed modules check cleanly
|
||||
# (CLAUDE.md: add pandas-stubs when introducing pandas to a module).
|
||||
pyright==1.1.411
|
||||
pandas-stubs
|
||||
|
|
|
|||
|
|
@ -92,7 +92,8 @@ from repositories.comparable_properties.epc_comparable_properties_repository imp
|
|||
EpcComparablePropertiesRepository,
|
||||
SkippedCohortCert,
|
||||
)
|
||||
from repositories.epc.epc_postgres_repository import EpcPostgresRepository
|
||||
from repositories.epc.epc_postgres_repository import EpcPostgresRepository, EpcSaveRequest
|
||||
from repositories.plan.plan_repository import PlanSaveRequest
|
||||
from repositories.geospatial.geospatial_s3_repository import (
|
||||
GeospatialS3Repository,
|
||||
ParquetReader,
|
||||
|
|
@ -178,31 +179,41 @@ class _PropertyWrite:
|
|||
def _flush_writes(engine: Engine, writes: list[_PropertyWrite]) -> None:
|
||||
"""Persist a whole batch of modelled Properties in one Unit of Work.
|
||||
|
||||
Replays each Property's saves in dependency order (EPC → spatial → solar →
|
||||
Plan → mark-modelled) and commits once. All-or-nothing per batch: a failed
|
||||
save rolls the whole transaction back and propagates, so the SQS message is
|
||||
retried — every save is an idempotent upsert, so a retry is safe. This mirrors
|
||||
the PropertyBaselineOrchestrator's existing one-UoW-per-batch contract
|
||||
(ADR-0012); per-property failures are isolated earlier, in the modelling loop,
|
||||
EPC writes are batched by source (lodged group first, predicted group second)
|
||||
so each source emits one DELETE pass + one INSERT pass regardless of batch
|
||||
size, rather than N×per-property round-trips (ADR-0012). All other writes
|
||||
(spatial, solar, plan, mark-modelled) remain per-property inside the same
|
||||
transaction. All-or-nothing per batch: a failed save rolls the whole
|
||||
transaction back so the SQS message is retried — every save is an idempotent
|
||||
upsert. Per-property failures are isolated earlier, in the modelling loop,
|
||||
before a write is ever queued."""
|
||||
lodged_requests = [
|
||||
EpcSaveRequest(w.lodged_epc, property_id=w.property_id, portfolio_id=w.portfolio_id, source="lodged")
|
||||
for w in writes
|
||||
if w.lodged_epc is not None and w.lodged_epc_is_new
|
||||
]
|
||||
predicted_requests = [
|
||||
EpcSaveRequest(w.predicted_epc, property_id=w.property_id, portfolio_id=w.portfolio_id, source="predicted")
|
||||
for w in writes
|
||||
if w.predicted_epc is not None and w.predicted_epc_is_new
|
||||
]
|
||||
with PostgresUnitOfWork(lambda: Session(engine)) as uow:
|
||||
if lodged_requests:
|
||||
uow.epc.save_batch(lodged_requests)
|
||||
if predicted_requests:
|
||||
uow.epc.save_batch(predicted_requests)
|
||||
plan_requests = [
|
||||
PlanSaveRequest(
|
||||
w.plan,
|
||||
property_id=w.property_id,
|
||||
scenario_id=w.scenario_id,
|
||||
portfolio_id=w.portfolio_id,
|
||||
is_default=w.is_default,
|
||||
)
|
||||
for w in writes
|
||||
]
|
||||
uow.plan.save_batch(plan_requests)
|
||||
for w in writes:
|
||||
if w.lodged_epc is not None and w.lodged_epc_is_new:
|
||||
uow.epc.save(
|
||||
w.lodged_epc,
|
||||
property_id=w.property_id,
|
||||
portfolio_id=w.portfolio_id,
|
||||
)
|
||||
elif w.predicted_epc is not None and w.predicted_epc_is_new:
|
||||
# Persist the synthesised EPC in the predicted slot (ADR-0031), so
|
||||
# the Baseline stage can re-hydrate it and downstream sees the
|
||||
# picture the Plan was modelled from.
|
||||
uow.epc.save(
|
||||
w.predicted_epc,
|
||||
property_id=w.property_id,
|
||||
portfolio_id=w.portfolio_id,
|
||||
source="predicted",
|
||||
)
|
||||
if w.spatial is not None:
|
||||
uow.spatial.save(w.uprn, w.spatial)
|
||||
if w.solar is not None:
|
||||
|
|
@ -212,13 +223,6 @@ def _flush_writes(engine: Engine, writes: list[_PropertyWrite]) -> None:
|
|||
latitude=w.solar.latitude,
|
||||
insights=w.solar.insights,
|
||||
)
|
||||
uow.plan.save(
|
||||
w.plan,
|
||||
property_id=w.property_id,
|
||||
scenario_id=w.scenario_id,
|
||||
portfolio_id=w.portfolio_id,
|
||||
is_default=w.is_default,
|
||||
)
|
||||
uow.property.mark_modelled(
|
||||
w.property_id, has_recommendations=w.has_recommendations
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6234,6 +6234,10 @@ _SAP_DESIGN_HEAT_LOSS_DELTA_T_K: Final[float] = 24.2
|
|||
_HP_SPACE_HEATING_IN_USE_FACTOR_N3_6: Final[float] = 0.95
|
||||
_HP_IN_USE_FACTOR_CRITERIA_MET: Final[float] = 0.95
|
||||
_HP_IN_USE_FACTOR_CRITERIA_FAIL: Final[float] = 0.60
|
||||
# SAP 10.2 Appendix N3.7 (PDF p.109): the heat-pump water-heating efficiency
|
||||
# (in-use factor × η_water) is "subject to a minimum efficiency of 100%" —
|
||||
# below that the direct-electric backup governs.
|
||||
_HP_WATER_HEATING_MIN_EFFICIENCY: Final[float] = 1.0
|
||||
|
||||
|
||||
def _heat_pump_cylinder_meets_pcdb_criteria(
|
||||
|
|
@ -6325,7 +6329,12 @@ def _heat_pump_apm_efficiencies(
|
|||
main_heating_efficiency = (
|
||||
_HP_SPACE_HEATING_IN_USE_FACTOR_N3_6 * eta_space_1_pct / 100.0
|
||||
)
|
||||
water_efficiency_pct = in_use_water * eta_water_3_pct / 100.0
|
||||
# N3.7: in-use factor × η_water, subject to a minimum efficiency of 100%
|
||||
# (the direct-electric backup floors the heat pump's water heating).
|
||||
water_efficiency_pct = max(
|
||||
in_use_water * eta_water_3_pct / 100.0,
|
||||
_HP_WATER_HEATING_MIN_EFFICIENCY,
|
||||
)
|
||||
return (main_heating_efficiency, water_efficiency_pct)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -263,6 +263,14 @@ _HP_PSR_GROUP_OFFSET_PSR: Final[int] = 0
|
|||
_HP_PSR_GROUP_OFFSET_ETA_SPACE_1: Final[int] = 2
|
||||
_HP_PSR_GROUP_OFFSET_ETA_WATER_3: Final[int] = 6
|
||||
|
||||
# SAP 10.2 Appendix N2 (PDF p.101, footnotes 44/45): out-of-range PSR
|
||||
# extension for air/ground/water source heat pumps. Above the record's
|
||||
# largest PSR the efficiency is reciprocal-interpolated toward 100% at
|
||||
# `_EXTENSION_PSR_MULTIPLE` × the largest PSR; below the smallest PSR, and
|
||||
# beyond that multiple, the efficiency is the terminal 100%.
|
||||
_EXTENSION_TERMINAL_EFFICIENCY_PCT: Final[float] = 100.0
|
||||
_EXTENSION_PSR_MULTIPLE: Final[float] = 2.0
|
||||
|
||||
|
||||
def _parse_psr_groups(raw: tuple[str, ...]) -> tuple[PsrEfficiencyGroup, ...]:
|
||||
"""Decode the variable-length PSR-dependent block of a format-465
|
||||
|
|
@ -317,28 +325,60 @@ def interpolate_heat_pump_efficiency_at_psr(
|
|||
(not their reciprocals taken from PCDB), so the η_*_pct values must
|
||||
be strictly positive — every PCDB row in the cohort satisfies this.
|
||||
|
||||
Per spec PDF p.100 lines 7039-7072: clamp to the smallest PSR in
|
||||
the record when `target_psr` is below it, and to the largest when
|
||||
above ("if the PSR is greater than the largest PSR in the database
|
||||
record then the heat pump space and water heating fractions for the
|
||||
largest PSR should be used, and if the PSR is less than the
|
||||
smallest PSR in the database record then the heat pump space and
|
||||
water heating fractions for the smallest PSR should be used").
|
||||
Out-of-range PSR (spec PDF p.101, footnotes 44/45 — air/ground/water
|
||||
source heat pumps):
|
||||
|
||||
- Below the smallest PSR in the record: "an efficiency of 100%
|
||||
should be used if the PSR is less than the smallest value in the
|
||||
database record."
|
||||
- Above the largest PSR in the record: "an efficiency may be
|
||||
obtained from linear interpolation between that at the largest
|
||||
PSR in the data record and efficiency 100% at PSR two times the
|
||||
largest PSR in the data record. If the PSR is greater than two
|
||||
times the largest PSR in the data record an efficiency of 100%
|
||||
should be used." The interpolation is reciprocal-linear too
|
||||
(footnote 43), with 100% as the upper anchor.
|
||||
|
||||
Both space- and water-heating PSR-dependent efficiencies extend the
|
||||
same way. (Exhaust-air heat pumps and combined heat-pump-and-boiler
|
||||
packages instead use 100% directly above the largest PSR, and combined
|
||||
packages clamp to the edge rows; neither is distinguished by the
|
||||
current PCDB parse, so the air/ground/water rule is applied uniformly
|
||||
— a documented limitation. The dominant RdSAP cohort is air source.)
|
||||
|
||||
Cohort fixture: cert 3336-2825-9400-0512-8292 (Mitsubishi PUZ-WM50VHA,
|
||||
PCDB 104568) — PSR 1.40151 brackets PCDB rows PSR 1.2 (η_space_1
|
||||
= 253.9) and PSR 1.5 (η_space_1 = 229.2). Linear (pre-slice):
|
||||
237.31; reciprocal (spec-faithful): 236.74 — matches worksheet
|
||||
(206)/(210) at 1e-4 once the 0.95 in-use factor is applied.
|
||||
|
||||
Out-of-range anchor: PCDB 100061 (golden fixture case 56), largest PSR
|
||||
2.0 (η_space_1=352.0). At dwelling PSR 3.10665 the extension to 100%
|
||||
at PSR 4.0 gives η_space_1 = 147.011 → (206) = 139.660, matching the
|
||||
accredited Elmhurst worksheet (vs the old clamp's 352.0 → 334.4%).
|
||||
"""
|
||||
if not psr_groups:
|
||||
raise ValueError("PSR groups required for interpolation")
|
||||
if target_psr <= psr_groups[0].psr:
|
||||
first = psr_groups[0]
|
||||
return (first.eta_space_1_pct, first.eta_water_3_pct)
|
||||
if target_psr >= psr_groups[-1].psr:
|
||||
if target_psr < psr_groups[0].psr:
|
||||
return (_EXTENSION_TERMINAL_EFFICIENCY_PCT, _EXTENSION_TERMINAL_EFFICIENCY_PCT)
|
||||
if target_psr > psr_groups[-1].psr:
|
||||
last = psr_groups[-1]
|
||||
return (last.eta_space_1_pct, last.eta_water_3_pct)
|
||||
upper_psr = _EXTENSION_PSR_MULTIPLE * last.psr
|
||||
if target_psr >= upper_psr:
|
||||
return (
|
||||
_EXTENSION_TERMINAL_EFFICIENCY_PCT,
|
||||
_EXTENSION_TERMINAL_EFFICIENCY_PCT,
|
||||
)
|
||||
t = (target_psr - last.psr) / (upper_psr - last.psr)
|
||||
eta_space_1 = 1.0 / (
|
||||
(1.0 - t) / last.eta_space_1_pct
|
||||
+ t / _EXTENSION_TERMINAL_EFFICIENCY_PCT
|
||||
)
|
||||
eta_water_3 = 1.0 / (
|
||||
(1.0 - t) / last.eta_water_3_pct
|
||||
+ t / _EXTENSION_TERMINAL_EFFICIENCY_PCT
|
||||
)
|
||||
return (eta_space_1, eta_water_3)
|
||||
for low_group, high_group in zip(psr_groups, psr_groups[1:]):
|
||||
if low_group.psr <= target_psr <= high_group.psr:
|
||||
span = high_group.psr - low_group.psr
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from typing import Optional, Protocol, TypeVar
|
||||
from typing import Any, Optional, Protocol, TypeVar
|
||||
|
||||
from sqlmodel import Session, col, delete, select
|
||||
from sqlalchemy import insert as _sa_insert
|
||||
from sqlmodel import Session, SQLModel, col, delete, select
|
||||
|
||||
from datatypes.epc.domain.epc import Epc
|
||||
from datatypes.epc.domain.epc_property_data import (
|
||||
|
|
@ -54,6 +56,23 @@ from utilities.private import private
|
|||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EpcSaveRequest:
|
||||
data: EpcPropertyData
|
||||
property_id: Optional[int] = None
|
||||
portfolio_id: Optional[int] = None
|
||||
source: EpcSource = field(default="lodged")
|
||||
|
||||
|
||||
def _col_values(model: SQLModel, exclude: frozenset[str] = frozenset()) -> dict[str, Any]:
|
||||
"""Extract column-keyed values from a SQLModel instance for Core INSERT."""
|
||||
return {
|
||||
c.name: getattr(model, c.name) # type: ignore[union-attr]
|
||||
for c in model.__table__.c # type: ignore[attr-defined]
|
||||
if c.name not in exclude # type: ignore[union-attr]
|
||||
}
|
||||
|
||||
|
||||
def _require(value: Optional[_T], field: str) -> _T:
|
||||
if value is None:
|
||||
raise ValueError(f"epc_property row is missing required field {field!r}")
|
||||
|
|
@ -111,75 +130,195 @@ class EpcPostgresRepository(EpcRepository):
|
|||
portfolio_id: Optional[int] = None,
|
||||
source: EpcSource = "lodged",
|
||||
) -> int:
|
||||
# Idempotent on (property_id, source): a re-run replaces the property's
|
||||
# EPC graph for THAT source rather than duplicating it (ADR-0012), and a
|
||||
# predicted save leaves the lodged one intact, and vice versa (ADR-0031).
|
||||
# Anonymous saves (no property_id) always insert.
|
||||
if property_id is not None:
|
||||
self._delete_for_property(property_id, source)
|
||||
parent = EpcPropertyModel.from_epc_property_data(
|
||||
data, property_id=property_id, portfolio_id=portfolio_id, source=source
|
||||
)
|
||||
self._session.add(parent)
|
||||
self._session.flush()
|
||||
epc_property_id = _require(parent.id, "id")
|
||||
return self.save_batch([EpcSaveRequest(data, property_id, portfolio_id, source)])[0]
|
||||
|
||||
self._session.add(
|
||||
EpcPropertyEnergyPerformanceModel.from_epc_property_data(
|
||||
data, epc_property_id=epc_property_id
|
||||
)
|
||||
)
|
||||
for detail in data.sap_heating.main_heating_details:
|
||||
self._session.add(
|
||||
EpcMainHeatingDetailModel.from_domain(detail, epc_property_id)
|
||||
)
|
||||
for part in data.sap_building_parts:
|
||||
bp = EpcBuildingPartModel.from_domain(part, epc_property_id)
|
||||
self._session.add(bp)
|
||||
self._session.flush()
|
||||
bp_id = _require(bp.id, "epc_building_part.id")
|
||||
for dim in part.sap_floor_dimensions:
|
||||
self._session.add(EpcFloorDimensionModel.from_domain(dim, bp_id))
|
||||
for window in data.sap_windows:
|
||||
self._session.add(EpcWindowModel.from_domain(window, epc_property_id))
|
||||
for index, array in enumerate(data.sap_energy_source.photovoltaic_arrays or []):
|
||||
self._session.add(
|
||||
EpcPhotovoltaicArrayModel.from_domain(array, index, epc_property_id)
|
||||
)
|
||||
def save_batch(self, requests: list[EpcSaveRequest]) -> list[int]:
|
||||
"""Insert all EPCs in `requests` in one pass per table, returning one
|
||||
epc_property_id per request in the same order as the input.
|
||||
|
||||
for element_type, elements in (
|
||||
("roof", data.roofs),
|
||||
("wall", data.walls),
|
||||
("floor", data.floors),
|
||||
("main_heating", data.main_heating),
|
||||
Deletes are batched first (one IN-query per child table per source),
|
||||
then all parent rows are inserted with a single RETURNING statement so
|
||||
positional ordering maps each returned id to its request. Building-part
|
||||
ids are captured the same way so floor-dimension FKs are resolved without
|
||||
any per-property flush round-trips (ADR-0012).
|
||||
"""
|
||||
if not requests:
|
||||
return []
|
||||
|
||||
# Batch-delete existing rows grouped by source so the lodged and predicted
|
||||
# slots remain independent (ADR-0031).
|
||||
pids_by_source: dict[EpcSource, list[int]] = {}
|
||||
for r in requests:
|
||||
if r.property_id is not None:
|
||||
pids_by_source.setdefault(r.source, []).append(r.property_id)
|
||||
for src, pids in pids_by_source.items():
|
||||
self._delete_for_properties(pids, src)
|
||||
|
||||
# Insert all parent (epc_property) rows; capture returned ids positionally.
|
||||
parent_rows = [
|
||||
_col_values(
|
||||
EpcPropertyModel.from_epc_property_data(
|
||||
r.data, property_id=r.property_id, portfolio_id=r.portfolio_id, source=r.source
|
||||
),
|
||||
exclude=frozenset({"id"}),
|
||||
)
|
||||
for r in requests
|
||||
]
|
||||
returned_parents = self._session.execute( # type: ignore[deprecated]
|
||||
_sa_insert(EpcPropertyModel).returning(EpcPropertyModel.__table__.c["id"]), # type: ignore[attr-defined]
|
||||
parent_rows,
|
||||
).all()
|
||||
epc_property_ids = [row[0] for row in returned_parents]
|
||||
|
||||
# Collect child rows, accumulating building parts in an ordered list so
|
||||
# the positional RETURNING trick can map part objects to their new ids.
|
||||
perf_rows: list[dict[str, Any]] = []
|
||||
heating_rows: list[dict[str, Any]] = []
|
||||
parts_ordered: list[tuple[Any, int]] = [] # (SapBuildingPart, epc_property_id)
|
||||
window_rows: list[dict[str, Any]] = []
|
||||
pv_rows: list[dict[str, Any]] = []
|
||||
element_rows: list[dict[str, Any]] = []
|
||||
flat_rows: list[dict[str, Any]] = []
|
||||
rhi_rows: list[dict[str, Any]] = []
|
||||
|
||||
for r, epc_pid in zip(requests, epc_property_ids):
|
||||
d = r.data
|
||||
perf_rows.append(
|
||||
_col_values(
|
||||
EpcPropertyEnergyPerformanceModel.from_epc_property_data(d, epc_pid),
|
||||
exclude=frozenset({"id"}),
|
||||
)
|
||||
)
|
||||
for detail in d.sap_heating.main_heating_details:
|
||||
heating_rows.append(
|
||||
_col_values(EpcMainHeatingDetailModel.from_domain(detail, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
for part in d.sap_building_parts:
|
||||
parts_ordered.append((part, epc_pid))
|
||||
for window in d.sap_windows:
|
||||
window_rows.append(
|
||||
_col_values(EpcWindowModel.from_domain(window, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
for idx, array in enumerate(d.sap_energy_source.photovoltaic_arrays or []):
|
||||
pv_rows.append(
|
||||
_col_values(EpcPhotovoltaicArrayModel.from_domain(array, idx, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
for etype, els in (
|
||||
("roof", d.roofs),
|
||||
("wall", d.walls),
|
||||
("floor", d.floors),
|
||||
("main_heating", d.main_heating),
|
||||
):
|
||||
for el in els:
|
||||
element_rows.append(
|
||||
_col_values(EpcEnergyElementModel.from_domain(el, etype, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
for el, etype in (
|
||||
(d.window, "window"),
|
||||
(d.lighting, "lighting"),
|
||||
(d.hot_water, "hot_water"),
|
||||
(d.secondary_heating, "secondary_heating"),
|
||||
(d.main_heating_controls, "main_heating_controls"),
|
||||
):
|
||||
if el is not None:
|
||||
element_rows.append(
|
||||
_col_values(EpcEnergyElementModel.from_domain(el, etype, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
if d.sap_flat_details is not None:
|
||||
flat_rows.append(
|
||||
_col_values(EpcFlatDetailsModel.from_domain(d.sap_flat_details, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
if d.renewable_heat_incentive is not None:
|
||||
rhi_rows.append(
|
||||
_col_values(EpcRenewableHeatIncentiveModel.from_domain(d.renewable_heat_incentive, epc_pid), frozenset({"id"}))
|
||||
)
|
||||
|
||||
# Bulk-insert all simple child tables (no downstream FK dependency).
|
||||
if perf_rows:
|
||||
self._session.execute(_sa_insert(EpcPropertyEnergyPerformanceModel), perf_rows) # type: ignore[deprecated]
|
||||
if heating_rows:
|
||||
self._session.execute(_sa_insert(EpcMainHeatingDetailModel), heating_rows) # type: ignore[deprecated]
|
||||
if window_rows:
|
||||
self._session.execute(_sa_insert(EpcWindowModel), window_rows) # type: ignore[deprecated]
|
||||
if pv_rows:
|
||||
self._session.execute(_sa_insert(EpcPhotovoltaicArrayModel), pv_rows) # type: ignore[deprecated]
|
||||
if element_rows:
|
||||
self._session.execute(_sa_insert(EpcEnergyElementModel), element_rows) # type: ignore[deprecated]
|
||||
if flat_rows:
|
||||
self._session.execute(_sa_insert(EpcFlatDetailsModel), flat_rows) # type: ignore[deprecated]
|
||||
if rhi_rows:
|
||||
self._session.execute(_sa_insert(EpcRenewableHeatIncentiveModel), rhi_rows) # type: ignore[deprecated]
|
||||
|
||||
# Building parts: insert with RETURNING and zip positionally to resolve
|
||||
# floor-dimension FKs. Do NOT key by id(part) — the same EpcPropertyData
|
||||
# object can appear in multiple requests (same epc, different property_ids),
|
||||
# giving identical object ids that collapse the dict and mis-wire FKs.
|
||||
# Positional zip is safe because PostgreSQL preserves VALUES order in RETURNING.
|
||||
if parts_ordered:
|
||||
bp_rows = [
|
||||
_col_values(EpcBuildingPartModel.from_domain(part, epc_pid), frozenset({"id"}))
|
||||
for part, epc_pid in parts_ordered
|
||||
]
|
||||
returned_bps = self._session.execute( # type: ignore[deprecated]
|
||||
_sa_insert(EpcBuildingPartModel).returning(EpcBuildingPartModel.__table__.c["id"]), # type: ignore[attr-defined]
|
||||
bp_rows,
|
||||
).all()
|
||||
floor_rows: list[dict[str, Any]] = [
|
||||
_col_values(EpcFloorDimensionModel.from_domain(dim, bp_row[0]), frozenset({"id"}))
|
||||
for (part, _), bp_row in zip(parts_ordered, returned_bps)
|
||||
for dim in part.sap_floor_dimensions
|
||||
]
|
||||
if floor_rows:
|
||||
self._session.execute(_sa_insert(EpcFloorDimensionModel), floor_rows) # type: ignore[deprecated]
|
||||
|
||||
return epc_property_ids
|
||||
|
||||
def _delete_for_properties(self, property_ids: list[int], source: EpcSource) -> None:
|
||||
"""Batch-delete every EPC graph for the given property_ids and source in
|
||||
one pass per child table (IN queries), replacing the per-property flush
|
||||
loop that drove RDS CPU to saturation during bulk modelling runs."""
|
||||
epc_ids = [
|
||||
i
|
||||
for i in self._session.exec(
|
||||
select(EpcPropertyModel.id)
|
||||
.where(col(EpcPropertyModel.property_id).in_(property_ids))
|
||||
.where(EpcPropertyModel.source == source)
|
||||
).all()
|
||||
if i is not None
|
||||
]
|
||||
if not epc_ids:
|
||||
return
|
||||
part_ids = [
|
||||
i
|
||||
for i in self._session.exec(
|
||||
select(EpcBuildingPartModel.id).where(
|
||||
col(EpcBuildingPartModel.epc_property_id).in_(epc_ids)
|
||||
)
|
||||
).all()
|
||||
if i is not None
|
||||
]
|
||||
if part_ids:
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcFloorDimensionModel).where(
|
||||
col(EpcFloorDimensionModel.epc_building_part_id).in_(part_ids)
|
||||
)
|
||||
)
|
||||
for child in (
|
||||
EpcPropertyEnergyPerformanceModel,
|
||||
EpcEnergyElementModel,
|
||||
EpcMainHeatingDetailModel,
|
||||
EpcBuildingPartModel,
|
||||
EpcWindowModel,
|
||||
EpcPhotovoltaicArrayModel,
|
||||
EpcFlatDetailsModel,
|
||||
EpcRenewableHeatIncentiveModel,
|
||||
):
|
||||
for el in elements:
|
||||
self._session.add(
|
||||
EpcEnergyElementModel.from_domain(el, element_type, epc_property_id)
|
||||
)
|
||||
for el, element_type in (
|
||||
(data.window, "window"),
|
||||
(data.lighting, "lighting"),
|
||||
(data.hot_water, "hot_water"),
|
||||
(data.secondary_heating, "secondary_heating"),
|
||||
(data.main_heating_controls, "main_heating_controls"),
|
||||
):
|
||||
if el is not None:
|
||||
self._session.add(
|
||||
EpcEnergyElementModel.from_domain(el, element_type, epc_property_id)
|
||||
)
|
||||
|
||||
if data.sap_flat_details is not None:
|
||||
self._session.add(
|
||||
EpcFlatDetailsModel.from_domain(data.sap_flat_details, epc_property_id)
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(child).where(col(child.epc_property_id).in_(epc_ids))
|
||||
)
|
||||
if data.renewable_heat_incentive is not None:
|
||||
self._session.add(
|
||||
EpcRenewableHeatIncentiveModel.from_domain(
|
||||
data.renewable_heat_incentive, epc_property_id
|
||||
)
|
||||
)
|
||||
return epc_property_id
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
delete(EpcPropertyModel).where(col(EpcPropertyModel.id).in_(epc_ids))
|
||||
)
|
||||
|
||||
def _delete_for_property(self, property_id: int, source: EpcSource) -> None:
|
||||
"""Remove the property's existing EPC graph for `source` (parent + child
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Literal, Optional
|
||||
from typing import TYPE_CHECKING, Literal, Optional
|
||||
|
||||
from datatypes.epc.domain.epc_property_data import EpcPropertyData
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from repositories.epc.epc_postgres_repository import EpcSaveRequest
|
||||
|
||||
# Provenance of a persisted EPC picture (ADR-0031): a real "lodged" EPC, or a
|
||||
# "predicted" one synthesised by EPC Prediction. A property can hold one of each.
|
||||
EpcSource = Literal["lodged", "predicted"]
|
||||
|
|
@ -29,6 +32,9 @@ class EpcRepository(ABC):
|
|||
source: EpcSource = "lodged",
|
||||
) -> int: ...
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(self, requests: "list[EpcSaveRequest]") -> list[int]: ...
|
||||
|
||||
@abstractmethod
|
||||
def get(self, epc_property_id: int) -> EpcPropertyData: ...
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,22 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import insert as _sa_insert
|
||||
from sqlmodel import Session, col, update
|
||||
|
||||
from domain.modelling.plan import Plan
|
||||
from infrastructure.postgres.modelling import PlanModel, RecommendationModel
|
||||
from repositories.plan.plan_repository import PlanRepository
|
||||
from repositories.plan.plan_repository import PlanRepository, PlanSaveRequest
|
||||
|
||||
|
||||
def _col_values(model: Any, exclude: frozenset[str] = frozenset()) -> dict[str, Any]:
|
||||
"""Extract column-keyed values from a SQLModel instance for Core INSERT."""
|
||||
return {
|
||||
c.name: getattr(model, c.name)
|
||||
for c in model.__table__.c
|
||||
if c.name not in exclude
|
||||
}
|
||||
|
||||
|
||||
class PlanPostgresRepository(PlanRepository):
|
||||
|
|
@ -29,37 +41,70 @@ class PlanPostgresRepository(PlanRepository):
|
|||
portfolio_id: int,
|
||||
is_default: bool,
|
||||
) -> int:
|
||||
# Soft-replace (ADR-0012): keep prior Plans as history rather than DELETEing
|
||||
# them — the cascade delete of recommendation rows was the slow part. When
|
||||
# this Plan is the default, demote every prior Plan for the same
|
||||
# (property_id, scenario_id) to is_default=False, so exactly one Plan for
|
||||
# the pair stays default (the one just inserted).
|
||||
if is_default:
|
||||
return self.save_batch(
|
||||
[PlanSaveRequest(plan, property_id=property_id, scenario_id=scenario_id, portfolio_id=portfolio_id, is_default=is_default)]
|
||||
)[0]
|
||||
|
||||
def save_batch(self, requests: list[PlanSaveRequest]) -> list[int]:
|
||||
"""Persist all Plans in three statements regardless of batch size.
|
||||
|
||||
1. One demote UPDATE (only when any request has ``is_default=True``).
|
||||
2. One bulk plan INSERT with RETURNING to capture ids positionally.
|
||||
3. One bulk recommendation INSERT (skipped when no measures exist).
|
||||
"""
|
||||
if not requests:
|
||||
return []
|
||||
|
||||
# Demote prior default Plans for every property in the batch that is
|
||||
# receiving a new default Plan — one UPDATE for the whole batch.
|
||||
default_pids = [r.property_id for r in requests if r.is_default]
|
||||
if default_pids:
|
||||
# scenario_id is uniform per batch (one scenario per SQS message).
|
||||
scenario_id = requests[0].scenario_id
|
||||
self._session.exec( # type: ignore[call-overload]
|
||||
update(PlanModel)
|
||||
.where(
|
||||
col(PlanModel.property_id) == property_id,
|
||||
col(PlanModel.property_id).in_(default_pids),
|
||||
col(PlanModel.scenario_id) == scenario_id,
|
||||
)
|
||||
.values(is_default=False)
|
||||
)
|
||||
|
||||
plan_row = PlanModel.from_domain(
|
||||
plan,
|
||||
property_id=property_id,
|
||||
scenario_id=scenario_id,
|
||||
portfolio_id=portfolio_id,
|
||||
is_default=is_default,
|
||||
)
|
||||
self._session.add(plan_row)
|
||||
self._session.flush()
|
||||
if plan_row.id is None:
|
||||
raise ValueError("plan row did not receive an id")
|
||||
|
||||
for measure in plan.measures:
|
||||
self._session.add(
|
||||
RecommendationModel.from_domain(
|
||||
measure, property_id=property_id, plan_id=plan_row.id
|
||||
)
|
||||
# Bulk INSERT all plan rows; capture returned ids positionally.
|
||||
plan_rows = [
|
||||
_col_values(
|
||||
PlanModel.from_domain(
|
||||
r.plan,
|
||||
property_id=r.property_id,
|
||||
scenario_id=r.scenario_id,
|
||||
portfolio_id=r.portfolio_id,
|
||||
is_default=r.is_default,
|
||||
),
|
||||
exclude=frozenset({"id"}),
|
||||
)
|
||||
return plan_row.id
|
||||
for r in requests
|
||||
]
|
||||
returned = self._session.execute( # type: ignore[deprecated]
|
||||
_sa_insert(PlanModel).returning(PlanModel.__table__.c["id"]), # type: ignore[attr-defined]
|
||||
plan_rows,
|
||||
).all()
|
||||
plan_ids = [row[0] for row in returned]
|
||||
|
||||
# Accumulate recommendation rows across all requests; properties with
|
||||
# zero measures contribute nothing (no special-casing needed).
|
||||
rec_rows = [
|
||||
_col_values(
|
||||
RecommendationModel.from_domain(
|
||||
measure, property_id=r.property_id, plan_id=plan_id
|
||||
),
|
||||
exclude=frozenset({"id"}),
|
||||
)
|
||||
for r, plan_id in zip(requests, plan_ids)
|
||||
for measure in r.plan.measures
|
||||
]
|
||||
if rec_rows:
|
||||
self._session.execute( # type: ignore[deprecated]
|
||||
_sa_insert(RecommendationModel), rec_rows
|
||||
)
|
||||
|
||||
return plan_ids
|
||||
|
|
|
|||
|
|
@ -1,10 +1,25 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
from domain.modelling.plan import Plan
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlanSaveRequest:
|
||||
"""Bundles the five fields the plan repository needs to persist one Plan.
|
||||
|
||||
Mirrors ``EpcSaveRequest`` in shape — used by ``PlanRepository.save_batch()``
|
||||
to accumulate write intent before the batch is flushed in one Unit of Work."""
|
||||
|
||||
plan: Plan
|
||||
property_id: int
|
||||
scenario_id: int
|
||||
portfolio_id: int
|
||||
is_default: bool
|
||||
|
||||
|
||||
class PlanRepository(ABC):
|
||||
"""Persists a Plan (and its Plan Measures) for a Property + Scenario.
|
||||
|
||||
|
|
@ -30,3 +45,12 @@ class PlanRepository(ABC):
|
|||
``(property_id, scenario_id)`` as history; when ``is_default`` is True,
|
||||
demotes those prior Plans to ``is_default=False``."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(self, requests: list[PlanSaveRequest]) -> list[int]:
|
||||
"""Persist a batch of Plans in three statements regardless of batch size.
|
||||
|
||||
Returns one plan id per request in input order. Fires a single demote
|
||||
UPDATE only when at least one request has ``is_default=True``. Keeps
|
||||
prior Plans as history (ADR-0017)."""
|
||||
...
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from applications.modelling_e2e.modelling_e2e_trigger_body import (
|
|||
ModellingE2ETriggerBody,
|
||||
)
|
||||
from domain.tasks.subtasks import SubTask
|
||||
from repositories.epc.epc_postgres_repository import EpcSaveRequest
|
||||
|
||||
PROPERTY_ID = 12345
|
||||
UPRN = 987654321
|
||||
|
|
@ -355,10 +356,10 @@ def test_lodged_epc_path_saves_epc_plan_and_marks_modelled(
|
|||
_call_handler(_BODY)
|
||||
|
||||
# Assert — EPC saved (lodged path), plan saved, property marked modelled
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
mock_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID
|
||||
mock_uow.epc.save_batch.assert_called_once_with(
|
||||
[EpcSaveRequest(mock_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID, source="lodged")]
|
||||
)
|
||||
mock_uow.plan.save.assert_called_once()
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
mock_uow.property.mark_modelled.assert_called_once_with(
|
||||
PROPERTY_ID, has_recommendations=False
|
||||
)
|
||||
|
|
@ -447,7 +448,7 @@ def test_skipped_cohort_certs_do_not_prevent_plan_being_saved() -> None:
|
|||
_call_handler(_BODY)
|
||||
|
||||
# Assert — plan committed despite the skipped cert
|
||||
mock_uow.plan.save.assert_called_once()
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
mock_uow.commit.assert_called_once()
|
||||
|
||||
|
||||
|
|
@ -512,7 +513,7 @@ def test_skipped_cohort_certs_are_logged_and_handler_does_not_raise() -> None:
|
|||
_call_handler(_BODY)
|
||||
|
||||
# Assert — plan committed; skipped cert number surfaced in a log call
|
||||
mock_uow.plan.save.assert_called_once()
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
mock_uow.commit.assert_called_once()
|
||||
logged_messages = " ".join(
|
||||
str(c.args) + str(c.kwargs) for c in mock_logger.info.call_args_list
|
||||
|
|
@ -631,13 +632,10 @@ def test_prediction_path_saves_predicted_epc_plan_and_baseline(
|
|||
_call_handler(_BODY)
|
||||
|
||||
# Assert — predicted EPC persisted in the predicted slot, plan saved, baseline run
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
mock_predicted_epc,
|
||||
property_id=PROPERTY_ID,
|
||||
portfolio_id=PORTFOLIO_ID,
|
||||
source="predicted",
|
||||
mock_uow.epc.save_batch.assert_called_once_with(
|
||||
[EpcSaveRequest(mock_predicted_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID, source="predicted")]
|
||||
)
|
||||
mock_uow.plan.save.assert_called_once()
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
mock_uow.commit.assert_called_once()
|
||||
_baseline_orchestrator.return_value.run.assert_called_once_with([PROPERTY_ID])
|
||||
|
||||
|
|
@ -841,13 +839,10 @@ def test_empty_own_postcode_broadens_to_nearby_and_predicts() -> None:
|
|||
# Assert — broadening fired, and the broadened cohort produced a saved plan
|
||||
# with its predicted EPC persisted in the predicted slot.
|
||||
MockRepo.return_value.candidates_near.assert_called_once()
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
mock_predicted_epc,
|
||||
property_id=PROPERTY_ID,
|
||||
portfolio_id=PORTFOLIO_ID,
|
||||
source="predicted",
|
||||
mock_uow.epc.save_batch.assert_called_once_with(
|
||||
[EpcSaveRequest(mock_predicted_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID, source="predicted")]
|
||||
)
|
||||
mock_uow.plan.save.assert_called_once()
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
mock_uow.commit.assert_called_once()
|
||||
|
||||
|
||||
|
|
@ -984,8 +979,9 @@ def test_batch_persists_in_one_transaction_and_one_baseline_run(
|
|||
"scenario_id": SCENARIO_ID, "refetch_solar": False, "dry_run": False}
|
||||
)
|
||||
|
||||
# Assert — all three Plans saved, but a single shared transaction:
|
||||
assert mock_uow.plan.save.call_count == 3
|
||||
# Assert — all three Plans saved in one batch call, but a single shared transaction:
|
||||
mock_uow.plan.save_batch.assert_called_once()
|
||||
assert len(mock_uow.plan.save_batch.call_args[0][0]) == 3
|
||||
assert mock_uow.property.mark_modelled.call_count == 3
|
||||
mock_uow.commit.assert_called_once()
|
||||
# One write Unit of Work opened for the whole batch, not one per property.
|
||||
|
|
@ -1170,10 +1166,8 @@ def test_refetch_epc_false_with_stored_epc_skips_api_call() -> None:
|
|||
# Assert — API not called; stored EPC flows into run_modelling
|
||||
mock_epc_client.get_by_uprn.assert_not_called()
|
||||
mock_run_modelling.assert_called_once()
|
||||
# Stored lodged EPC is persisted in the lodged slot
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
stored_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID
|
||||
)
|
||||
# Stored EPC is NOT re-saved — it was read from DB unchanged (PR #1353)
|
||||
mock_uow.epc.save_batch.assert_not_called()
|
||||
|
||||
|
||||
def test_refetch_epc_false_without_stored_epc_skips_api_and_goes_to_prediction() -> None:
|
||||
|
|
@ -1258,11 +1252,8 @@ def test_refetch_epc_false_without_stored_epc_skips_api_and_goes_to_prediction()
|
|||
|
||||
# Assert — API was NOT called; prediction ran and its output was persisted
|
||||
mock_epc_client.get_by_uprn.assert_not_called()
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
mock_predicted_epc,
|
||||
property_id=PROPERTY_ID,
|
||||
portfolio_id=PORTFOLIO_ID,
|
||||
source="predicted",
|
||||
mock_uow.epc.save_batch.assert_called_once_with(
|
||||
[EpcSaveRequest(mock_predicted_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID, source="predicted")]
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1396,14 +1387,9 @@ def test_repredict_epc_false_with_stored_predicted_epc_skips_prediction() -> Non
|
|||
# Act
|
||||
_call_handler({**_BODY, "repredict_epc": False})
|
||||
|
||||
# Assert — EpcPrediction.predict never called; stored EPC persisted in predicted slot
|
||||
# Assert — EpcPrediction.predict never called; stored predicted EPC NOT re-saved (PR #1353)
|
||||
mock_predictor.predict.assert_not_called()
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
stored_predicted,
|
||||
property_id=PROPERTY_ID,
|
||||
portfolio_id=PORTFOLIO_ID,
|
||||
source="predicted",
|
||||
)
|
||||
mock_uow.epc.save_batch.assert_not_called()
|
||||
|
||||
|
||||
def test_repredict_epc_false_without_stored_predicted_epc_falls_back_to_live_prediction() -> None:
|
||||
|
|
@ -1488,11 +1474,8 @@ def test_repredict_epc_false_without_stored_predicted_epc_falls_back_to_live_pre
|
|||
|
||||
# Assert — live prediction was used as fallback
|
||||
mock_predictor.predict.assert_called_once()
|
||||
mock_uow.epc.save.assert_called_once_with(
|
||||
mock_predicted_epc,
|
||||
property_id=PROPERTY_ID,
|
||||
portfolio_id=PORTFOLIO_ID,
|
||||
source="predicted",
|
||||
mock_uow.epc.save_batch.assert_called_once_with(
|
||||
[EpcSaveRequest(mock_predicted_epc, property_id=PROPERTY_ID, portfolio_id=PORTFOLIO_ID, source="predicted")]
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,15 @@ _FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "epc_prediction"
|
|||
# new-build-vs-old-stock service mismatch on 1-2 targets each (heating_main_fuel
|
||||
# 0.9722->0.9394, water_heating_fuel ->0.9495, cylinder_insulation_type 0.6667->
|
||||
# 0.3333) plus floor_area (+0.31 MAE). Tighten-only resumes from these values.
|
||||
#
|
||||
# has_pv re-baselined 0.9798->0.9697 when full-SAP lodged PV mapping landed
|
||||
# (datatypes/epc/domain/mapper.py `_sap_17_1_pv_arrays`): full-SAP certs lodge
|
||||
# their measured array under `sap_energy_source.pv_arrays`, which the schema
|
||||
# dropped at parse, so the leave-one-out scorer's *actual* has_pv read False for
|
||||
# every full-SAP PV dwelling. Carrying the array now reads the true has_pv=True,
|
||||
# and one full-SAP target the similarity-weighted donors don't predict as PV
|
||||
# tips the agreement 32/33 (the held-out actual is now correct — a ground-truth-
|
||||
# method change, not a prediction-logic loosening). Tighten-only resumes here.
|
||||
_RATE_FLOORS: dict[str, float] = {
|
||||
"wall_construction": 0.9091,
|
||||
"wall_insulation_type": 0.8687,
|
||||
|
|
@ -76,7 +85,7 @@ _RATE_FLOORS: dict[str, float] = {
|
|||
"floor_insulation": 0.9375,
|
||||
"has_room_in_roof": 0.9495,
|
||||
"modal_glazing_type": 0.8384,
|
||||
"has_pv": 0.9798,
|
||||
"has_pv": 0.9697,
|
||||
"solar_water_heating": 1.0000,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from domain.sap10_calculator.rdsap.cert_to_inputs import (
|
|||
_apply_heat_network_hiu_default_store, # pyright: ignore[reportPrivateUsage]
|
||||
_cylinder_thermostat_present, # pyright: ignore[reportPrivateUsage]
|
||||
_has_suspended_timber_floor_per_spec, # pyright: ignore[reportPrivateUsage]
|
||||
_heat_pump_apm_efficiencies, # pyright: ignore[reportPrivateUsage]
|
||||
_heat_network_code_302_effective_factor, # pyright: ignore[reportPrivateUsage]
|
||||
_heat_network_community_fuel_code, # pyright: ignore[reportPrivateUsage]
|
||||
_heat_network_distribution_electricity, # pyright: ignore[reportPrivateUsage]
|
||||
|
|
@ -4841,6 +4842,57 @@ def test_hot_water_from_pcdb_heat_pump_bills_at_app_n_wh_high_rate() -> None:
|
|||
assert abs(rate_immersion - 0.0750) <= 1e-6
|
||||
|
||||
|
||||
def test_heat_pump_water_efficiency_is_floored_at_100pct_per_app_n3_7() -> None:
|
||||
# Arrange — SAP 10.2 Appendix N3.7 ("Thermal efficiency for water
|
||||
# heating – heat pumps", PDF p.109): "multiply the thermal efficiency
|
||||
# (ηwater) for water heating by the in-use factor in Table N8; subject
|
||||
# to a MINIMUM EFFICIENCY OF 100%." Our `_heat_pump_apm_efficiencies`
|
||||
# applied the in-use factor but omitted the floor, so an oversized heat
|
||||
# pump whose PSR-extended ηwater × 0.60 in-use fell below 100% billed
|
||||
# water heating at that sub-100% efficiency (over-counting HW fuel).
|
||||
#
|
||||
# Accredited anchor: golden fixture case 56 (PCDB 100061, the config of
|
||||
# cert 100110101713). At HLC 107.82 W/K the PSR is 3.107, above the
|
||||
# record's largest PSR 2.0, so the Appendix N2 extension takes ηwater,3
|
||||
# from 198.9% toward 100% at 2 x 2.0 = 4.0 → 128.55%; × the 0.60 in-use
|
||||
# factor (Open-EPC certs never lodge cylinder HX area → criteria fail)
|
||||
# = 77.13% < 100% → the worksheet (216) reads 100.0000. In-range PSR
|
||||
# (case 54, HLC large) keeps 0.60 × 198.9 = 119.34% (worksheet case 54
|
||||
# (216) = 112.5% for its 187.5% record — both above the floor, unchanged).
|
||||
from domain.sap10_calculator.tables.pcdb import heat_pump_record
|
||||
|
||||
record = heat_pump_record(100061)
|
||||
assert record is not None
|
||||
hp_main = MainHeatingDetail(
|
||||
has_fghrs=False,
|
||||
main_fuel_type=29, # electricity (heat pump)
|
||||
heat_emitter_type=1, # radiators
|
||||
emitter_temperature=0,
|
||||
main_heating_control=2210,
|
||||
main_heating_category=4,
|
||||
sap_main_heating_code=None,
|
||||
main_heating_index_number=100061,
|
||||
)
|
||||
epc = _typical_semi_detached_epc() # no specified cylinder → in-use 0.60
|
||||
|
||||
# Act — oversized PSR (extension region) vs an in-range PSR.
|
||||
_space_ext, water_ext = _heat_pump_apm_efficiencies(
|
||||
main=hp_main, hp_record=record,
|
||||
hlc_annual_avg_w_per_k=107.82, # PSR 3.107 > largest 2.0
|
||||
epc=epc,
|
||||
) or (None, None)
|
||||
_space_in, water_in = _heat_pump_apm_efficiencies(
|
||||
main=hp_main, hp_record=record,
|
||||
hlc_annual_avg_w_per_k=400.0, # PSR 0.837, in range
|
||||
epc=epc,
|
||||
) or (None, None)
|
||||
|
||||
# Assert — extended HP water efficiency is floored at 100% (1.0); the
|
||||
# in-range PSR keeps the un-floored 0.60 × 198.9% = 119.34%.
|
||||
assert water_ext is not None and abs(water_ext - 1.0) < 1e-9
|
||||
assert water_in is not None and abs(water_in - 0.60 * 198.9 / 100.0) < 1e-9
|
||||
|
||||
|
||||
def test_hot_water_immersion_off_peak_bills_at_table_13_blend() -> None:
|
||||
# Arrange — SAP 10.2 Table 12a (PDF p.191) "Immersion water heater"
|
||||
# row routes the WH column to Table 13 (PDF p.197). For an electric
|
||||
|
|
|
|||
|
|
@ -180,3 +180,88 @@ def test_interpolate_heat_pump_efficiency_at_cert_0380_psr_per_sap_app_n() -> No
|
|||
# ≈ 0.0035077 → eta_water_3 ≈ 285.0861
|
||||
assert abs(eta_space_1 - 234.5235) < 1e-3
|
||||
assert abs(eta_water_3 - 285.0861) < 1e-3
|
||||
|
||||
|
||||
def test_interpolate_extends_above_largest_psr_toward_100pct_per_app_n() -> None:
|
||||
"""SAP 10.2 Appendix N2 (PDF p.101, footnote 44/45) — PSR above the
|
||||
record's largest value extends the efficiency toward 100%, it is NOT
|
||||
clamped to the top-of-table value.
|
||||
|
||||
"in the case of a heat pump (ground, water or air source), where
|
||||
the PSR is greater than the largest value in the data record, an
|
||||
efficiency may be obtained from linear interpolation between that
|
||||
at the largest PSR in the data record and efficiency 100% at PSR
|
||||
two times the largest PSR in the data record. If the PSR is
|
||||
greater than two times the largest PSR in the data record an
|
||||
efficiency of 100% should be used."
|
||||
|
||||
Interpolation is reciprocal-linear (footnote 43). Accredited anchor:
|
||||
Elmhurst worksheet for cert 100110101713 / "golden fixture debugging"
|
||||
case 56 (PCDB 100061, ECODAN 8.5 kW, largest PSR row η_space,1=352.0).
|
||||
The dwelling HLC (39) = 107.8199 W/K and max output 8.106 kW give
|
||||
|
||||
PSR = 8.106 × 1000 / (107.8199 × 24.2) = 3.106650
|
||||
|
||||
which exceeds the record's largest PSR (2.0). The spec extension to
|
||||
100% at 2 × 2.0 = 4.0 yields, at t = (3.106650 − 2.0)/(4.0 − 2.0):
|
||||
|
||||
1/η = (1 − t)/352.0 + t/100.0 → η_space,1 = 147.011
|
||||
|
||||
so that (206) = 0.95 × 147.011 = 139.660 — matching the accredited
|
||||
worksheet exactly. The previous top-of-table clamp returned 352.0
|
||||
(→ 334.4%), over-rating the dwelling by +18 SAP.
|
||||
"""
|
||||
from domain.sap10_calculator.tables.pcdb.parser import (
|
||||
interpolate_heat_pump_efficiency_at_psr,
|
||||
)
|
||||
|
||||
record = heat_pump_record(100061)
|
||||
assert record is not None
|
||||
assert record.psr_groups[-1].psr == 2.0
|
||||
assert record.psr_groups[-1].eta_space_1_pct == 352.0
|
||||
|
||||
eta_space_1, _eta_water_3 = interpolate_heat_pump_efficiency_at_psr(
|
||||
record.psr_groups, target_psr=3.106649864134083,
|
||||
)
|
||||
|
||||
assert abs(eta_space_1 - 147.011) < 1e-2
|
||||
assert abs(0.95 * eta_space_1 - 139.6604) < 1e-2
|
||||
|
||||
|
||||
def test_interpolate_above_twice_largest_psr_is_100pct_per_app_n() -> None:
|
||||
"""SAP 10.2 Appendix N2 — beyond twice the largest PSR the efficiency
|
||||
is exactly 100% (the upper terminus of the extension), for both space
|
||||
and water heating PSR-dependent results."""
|
||||
from domain.sap10_calculator.tables.pcdb.parser import (
|
||||
interpolate_heat_pump_efficiency_at_psr,
|
||||
)
|
||||
|
||||
record = heat_pump_record(100061)
|
||||
assert record is not None
|
||||
|
||||
eta_space_1, eta_water_3 = interpolate_heat_pump_efficiency_at_psr(
|
||||
record.psr_groups, target_psr=9.0, # > 2 × 2.0
|
||||
)
|
||||
|
||||
assert eta_space_1 == 100.0
|
||||
assert eta_water_3 == 100.0
|
||||
|
||||
|
||||
def test_interpolate_below_smallest_psr_is_100pct_per_app_n() -> None:
|
||||
"""SAP 10.2 Appendix N2 (PDF p.101) — "For all heat pumps, an
|
||||
efficiency of 100% should be used if the PSR is less than the smallest
|
||||
value in the database record." (Not clamped to the smallest row.)"""
|
||||
from domain.sap10_calculator.tables.pcdb.parser import (
|
||||
interpolate_heat_pump_efficiency_at_psr,
|
||||
)
|
||||
|
||||
record = heat_pump_record(100061)
|
||||
assert record is not None
|
||||
assert record.psr_groups[0].psr == 0.2
|
||||
|
||||
eta_space_1, eta_water_3 = interpolate_heat_pump_efficiency_at_psr(
|
||||
record.psr_groups, target_psr=0.1, # < 0.2
|
||||
)
|
||||
|
||||
assert eta_space_1 == 100.0
|
||||
assert eta_water_3 == 100.0
|
||||
|
|
|
|||
|
|
@ -122,12 +122,18 @@ _EXPECTATIONS: Final[tuple[RealCertExpectation, ...]] = (
|
|||
# (engine uses the cert's measured 0.19/0.11/0.11 U-values; Elmhurst uses
|
||||
# age-band L proxies + party-wall default) plus FGHRS (cert idx 60031) omitted
|
||||
# on BOTH sides (the engine can't yet model full-SAP FGHRS). PINNED TO THE
|
||||
# OBSERVED 82, not lodged 84 — mapping deliberately untuned.
|
||||
# OBSERVED 83 (was 82), not lodged 84 — mapping deliberately untuned.
|
||||
# WAS 82 until the full-SAP electricity-tariff → RdSAP meter_type fix: this
|
||||
# cert lodges energy_tariff=1 (standard), which the mapper previously passed
|
||||
# through untranslated as RdSAP meter_type "1" — wrongly read as dual/Economy 7
|
||||
# and priced on the off-peak high/low split. Translating it to "single" (the
|
||||
# correct standard tariff) re-prices its electricity at the flat rate, lifting
|
||||
# this gas semi 82→83. No PV (sap_energy_source.pv_arrays absent).
|
||||
RealCertExpectation(
|
||||
schema="SAP-Schema-17.1",
|
||||
sample="uprn_10093116528",
|
||||
cert_num="8000-8495-2839-2607-9683",
|
||||
sap_score=82,
|
||||
sap_score=83,
|
||||
),
|
||||
# UPRN 10093116543 → cert 8358-7436-5620-6889-0906. SAP-Schema-17.1 — a
|
||||
# FULL-SAP cert (2017 mains-gas COMBI semi, Emsworth), forced through the
|
||||
|
|
@ -290,13 +296,18 @@ _EXPECTATIONS: Final[tuple[RealCertExpectation, ...]] = (
|
|||
# control 2106 (CBE); water from primary (combi); MEV on; AP50 Blower Door 3.5.
|
||||
# The −3 vs lodged 85 is the documented full-SAP→RdSAP gap: the engine uses the
|
||||
# cert's MEASURED U (wall 0.24 / floor 0.13, WORSE than RdSAP band-M defaults)
|
||||
# + MEV priced as extract loss not heat recovery. PINNED to the observed 82 —
|
||||
# mapping untuned; engine == Elmhurst.
|
||||
# + MEV priced as extract loss not heat recovery. PINNED to the observed 84
|
||||
# (was 82), still −1 vs lodged 85 — mapping untuned.
|
||||
# WAS 82 until full-SAP lodged PV mapping landed: this cert lodges a 0.38 kWp
|
||||
# array under sap_energy_source.pv_arrays (SE-facing, pitch 30°, unshaded) that
|
||||
# the schema dropped at parse, so the Appendix-M generation credit was lost.
|
||||
# Carrying it (mapper `_sap_17_1_pv_arrays`) credits the generation and lifts
|
||||
# this flat 82→84, closing most of the gap to the lodged 85 the array explains.
|
||||
RealCertExpectation(
|
||||
schema="SAP-Schema-19.1.0",
|
||||
sample="uprn_10096028301",
|
||||
cert_num="0390-3321-6060-2405-7985",
|
||||
sap_score=82,
|
||||
sap_score=84,
|
||||
),
|
||||
# UPRN 44012843 → cert 0775-2898-6628-9594-8005. SAP-Schema-16.3 — a
|
||||
# reduced-field (RdSAP-shaped) ground-floor FLAT, band K (2007-2011), cavity
|
||||
|
|
@ -326,14 +337,20 @@ _EXPECTATIONS: Final[tuple[RealCertExpectation, ...]] = (
|
|||
# worksheet SAP 80 — engine EXACTLY matches (80.13 vs 80); engine-on-Elmhurst's-
|
||||
# own-parsed-inputs 81.03 ≈ 80 → calculator faithful. Boiler set to the cert's
|
||||
# exact PCDB 16211 via the search dialog; control 2106 (CBE); water from primary
|
||||
# (combi); MEV on; AP50 Blower Door 3.2; party wall 6.43 m entered. The −2 vs
|
||||
# lodged 82 is the documented full-SAP→RdSAP gap (measured U 0.2/0.1 + MEV
|
||||
# extract loss). PINNED to the observed 80 — mapping untuned; engine == Elmhurst.
|
||||
# (combi); MEV on; AP50 Blower Door 3.2; party wall 6.43 m entered.
|
||||
# WAS 80 (engine == Elmhurst, both built WITHOUT PV) until full-SAP lodged PV
|
||||
# mapping landed: this cert lodges a 0.48 kWp array under
|
||||
# sap_energy_source.pv_arrays (SE-facing, pitch 30°, unshaded) the schema
|
||||
# dropped at parse. Crediting it (mapper `_sap_17_1_pv_arrays`) closes the
|
||||
# −2 gap exactly — the engine now reproduces the accredited lodged 82. The
|
||||
# Elmhurst worksheet (80) omitted the PV (not entered in the RdSAP build), so
|
||||
# the +2 over Elmhurst is the now-credited array, not a calculator drift.
|
||||
# PINNED to the observed 82 == lodged 82 — mapping untuned.
|
||||
RealCertExpectation(
|
||||
schema="SAP-Schema-17.0",
|
||||
sample="uprn_10023444324",
|
||||
cert_num="8501-5064-6739-1407-0163",
|
||||
sap_score=80,
|
||||
sap_score=82,
|
||||
),
|
||||
# UPRN 10023444320 → cert 0868-6045-7331-4376-0914. SAP-Schema-17.0 — FULL-SAP
|
||||
# MID-FLOOR FLAT (sibling of 10023444324, same block / combi PCDB 16211 / MEV),
|
||||
|
|
@ -342,12 +359,24 @@ _EXPECTATIONS: Final[tuple[RealCertExpectation, ...]] = (
|
|||
# worksheet 82 — engine within ~1 (81.38 vs 82); engine-on-Elmhurst-inputs 82.46
|
||||
# ≈ 82 → calculator faithful. Boiler PCDB 16211 via search; control 2106 (CBE);
|
||||
# water from primary (combi); MEV on; AP50 Blower Door 3.09; mid-floor (floor =
|
||||
# another dwelling below). PINNED to the observed 81 — mapping untuned.
|
||||
# another dwelling below).
|
||||
# WAS 81 until full-SAP lodged PV mapping landed: this cert lodges the SAME
|
||||
# 0.48 kWp array as its ground-floor sibling 10023444324 under
|
||||
# sap_energy_source.pv_arrays (the block's roof PV apportioned to the flat on
|
||||
# the lodged cert). Crediting it faithfully (mapper `_sap_17_1_pv_arrays`)
|
||||
# lifts this flat 81→83. NOTE this lands +2 OVER the lodged 81 (and +1 over the
|
||||
# Elmhurst worksheet 82) — unlike the ground-floor sibling whose pre-PV engine
|
||||
# was 2 UNDER lodged so the same array closed the gap exactly. The mid-floor's
|
||||
# pre-PV engine already matched lodged, so the credited array now overshoots:
|
||||
# the lodged 81 does not appear to carry the array's full generation credit
|
||||
# that SAP Appendix-M awards it. This is the documented full-SAP→RdSAP residual
|
||||
# (faithful to the cert's lodged PV, not tuned to the lodged integer). PINNED
|
||||
# to the observed 83 — mapping untuned.
|
||||
RealCertExpectation(
|
||||
schema="SAP-Schema-17.0",
|
||||
sample="uprn_10023444320",
|
||||
cert_num="0868-6045-7331-4376-0914",
|
||||
sap_score=81,
|
||||
sap_score=83,
|
||||
),
|
||||
# UPRN 10090844932 → cert 0646-3008-6208-0619-6204. RdSAP-Schema-20.0.0 —
|
||||
# END-TERRACE HOUSE, 2-storey, band L (2012-2022), cavity insulated, pitched
|
||||
|
|
|
|||
|
|
@ -193,7 +193,10 @@ _CORPUS = Path(
|
|||
# within-0.5 71.6% -> 72.5%, MAE 0.819 -> 0.815. Surfaced by Khalim's Elmhurst
|
||||
# stress worksheet (simulated case 46): closed its last ventilation residual
|
||||
# (our Jan ACH 9.14 -> 9.0748 exact; SAP 29 -> 30 = accredited Elmhurst).
|
||||
_MIN_WITHIN_HALF_SAP = 0.74
|
||||
# 0.74 -> 0.742 via the heat-pump water-heating 100% floor (App N3.7): cert
|
||||
# 100110101713 moves inside +-0.5 (|err| 4.97 -> 0.49). See the _MAX_SAP_MAE
|
||||
# log below for the paired space-heating PSR-extension + water-floor slices.
|
||||
_MIN_WITHIN_HALF_SAP = 0.742
|
||||
# 0.793 -> 0.789 via the §12 Unknown-meter + dual-electric-immersion off-peak
|
||||
# trigger (RdSAP 10 PDF p.62): Apartment 241 (main 691 + 903 dual immersion)
|
||||
# -5.38 -> -1.05. Worksheet-validated on "simulated case 48" (Elmhurst SAP 57,
|
||||
|
|
@ -248,7 +251,22 @@ _MIN_WITHIN_HALF_SAP = 0.74
|
|||
# an identical dwelling rates SAP 87 with "Connected to Dwelling = Yes" (credit
|
||||
# -£167) vs SAP 74 with "No" (credit £0). Enum decoded empirically: 0 = no PV,
|
||||
# 1 = not connected, 2 = connected (the gov-API does not expose it elsewhere).
|
||||
_MAX_SAP_MAE = 0.740
|
||||
# Then 0.740 -> 0.726 via the heat-pump PSR-extension fix (SAP 10.2 Appendix N2,
|
||||
# PDF p.101 footnotes 44/45): an air/ground/water source heat pump whose plant
|
||||
# size ratio exceeds the PCDB record's largest PSR is no longer clamped to the
|
||||
# top-of-table COP — its efficiency is reciprocal-interpolated toward 100% at
|
||||
# twice the largest PSR (and 100% below the smallest PSR). Accredited Elmhurst
|
||||
# worksheet for cert 100110101713 (golden fixture case 56, PCDB 100061, PSR
|
||||
# 3.107 over largest 2.0): (206) 334.4% -> 139.66% = Elmhurst exact. Only two
|
||||
# certs move (both oversized-PSR heat pumps): 100110101713 +18.32 -> -4.97 and
|
||||
# 4510053280 -0.61; within-0.5 holds at 74.1%.
|
||||
# Then 0.726 -> 0.722 (within-0.5 74.1% -> 74.2%) via the heat-pump water-
|
||||
# heating 100% floor (SAP 10.2 Appendix N3.7, PDF p.109: in-use x eta_water
|
||||
# subject to a minimum efficiency of 100%). Only 100110101713 moves: its
|
||||
# oversized-PSR water eff 0.60 x 128.55% = 77.13% is floored to 100% (=
|
||||
# accredited Elmhurst (216)), taking the cert 68.03 -> 72.51 (|err| 4.97 ->
|
||||
# 0.49, now inside +-0.5). In-range heat pumps keep their > 100% water COP.
|
||||
_MAX_SAP_MAE = 0.722
|
||||
_MAX_CO2_MAE_TONNES = 0.09 # t CO2 / yr vs co2_emissions_current
|
||||
_MAX_PE_PER_M2_MAE = 3.5 # kWh / m2 / yr vs energy_consumption_current
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ from domain.modelling.scenario import Scenario
|
|||
from domain.property_baseline.property_baseline_performance import PropertyBaselinePerformance
|
||||
from domain.property.properties import Properties
|
||||
from domain.property.property import Property
|
||||
from repositories.plan.plan_repository import PlanRepository
|
||||
from repositories.epc.epc_postgres_repository import EpcSaveRequest
|
||||
from repositories.plan.plan_repository import PlanRepository, PlanSaveRequest
|
||||
from repositories.product.product_repository import ProductRepository
|
||||
from repositories.property_baseline.property_baseline_repository import PropertyBaselineRepository
|
||||
from repositories.epc.epc_repository import EpcRepository, EpcSource
|
||||
|
|
@ -130,6 +131,9 @@ class FakeEpcRepo(EpcRepository):
|
|||
if property_id in self._predicted_by_property
|
||||
}
|
||||
|
||||
def save_batch(self, requests: list[EpcSaveRequest]) -> list[int]:
|
||||
return [self.save(r.data, r.property_id, r.portfolio_id, r.source) for r in requests]
|
||||
|
||||
|
||||
class FakeSolarRepo(SolarRepository):
|
||||
"""In-memory Google Solar insights store keyed by UPRN. Seed `by_uprn` to
|
||||
|
|
@ -218,6 +222,18 @@ class FakePlanRepository(PlanRepository):
|
|||
self._next_id += 1
|
||||
return plan_id
|
||||
|
||||
def save_batch(self, requests: list[PlanSaveRequest]) -> list[int]:
|
||||
return [
|
||||
self.save(
|
||||
r.plan,
|
||||
property_id=r.property_id,
|
||||
scenario_id=r.scenario_id,
|
||||
portfolio_id=r.portfolio_id,
|
||||
is_default=r.is_default,
|
||||
)
|
||||
for r in requests
|
||||
]
|
||||
|
||||
|
||||
class _UnsetProductRepo(ProductRepository):
|
||||
"""Default for a `FakeUnitOfWork` built without a catalogue — raises if a
|
||||
|
|
|
|||
167
tests/repositories/epc/test_epc_batch_save.py
Normal file
167
tests/repositories/epc/test_epc_batch_save.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Batch EPC write path — save_batch() correctness and safety tests.
|
||||
|
||||
Guards the four user stories from #1348:
|
||||
1. FK mis-wiring regression: building-part IDs must not be crossed between
|
||||
properties in the same save_batch() call.
|
||||
2. save()/save_batch() parity: the single-property delegation path is loss-free.
|
||||
3. Batch idempotency: a second save_batch() with the same requests replaces,
|
||||
not duplicates.
|
||||
4. Source isolation: lodged and predicted slots coexist after separate
|
||||
save_batch() calls on the same property IDs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Engine
|
||||
from sqlmodel import Session
|
||||
|
||||
from datatypes.epc.domain.epc_property_data import EpcPropertyData
|
||||
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
|
||||
from repositories.epc.epc_postgres_repository import EpcPostgresRepository, EpcSaveRequest
|
||||
|
||||
_JSON_SAMPLES = Path(__file__).resolve().parents[3] / "backend/epc_api/json_samples"
|
||||
|
||||
|
||||
def _load_epc(schema_dir: str = "RdSAP-Schema-21.0.0") -> EpcPropertyData:
|
||||
raw: dict[str, Any] = json.loads(
|
||||
(_JSON_SAMPLES / schema_dir / "epc.json").read_text()
|
||||
)
|
||||
return EpcPropertyDataMapper.from_api_response(raw)
|
||||
|
||||
|
||||
def _with_floor_areas(epc: EpcPropertyData, areas_m2: list[float]) -> EpcPropertyData:
|
||||
"""Replace the building parts with variants that have a single floor dimension
|
||||
carrying the given total_floor_area_m2 — making them easy to distinguish after
|
||||
a round-trip without changing anything else about the EPC."""
|
||||
template_bp = epc.sap_building_parts[0]
|
||||
template_dim = template_bp.sap_floor_dimensions[0]
|
||||
new_parts = [
|
||||
replace(template_bp, sap_floor_dimensions=[replace(template_dim, total_floor_area_m2=a)])
|
||||
for a in areas_m2
|
||||
]
|
||||
return replace(epc, sap_building_parts=new_parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tracer bullet: single-request save_batch() is loss-free vs save()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_single_request_save_batch_matches_save(db_engine: Engine) -> None:
|
||||
# Arrange
|
||||
epc = _load_epc()
|
||||
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
epc_id_via_save = repo.save(epc, property_id=1001)
|
||||
epc_id_via_batch = repo.save_batch([EpcSaveRequest(epc, property_id=1002)])[0]
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
via_save = repo.get(epc_id_via_save)
|
||||
via_batch = repo.get(epc_id_via_batch)
|
||||
|
||||
# Assert — both paths reconstruct the original exactly.
|
||||
assert via_save == epc
|
||||
assert via_batch == epc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FK mis-wiring regression: building-part IDs must not be crossed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_multi_property_building_part_ids_are_not_crossed(db_engine: Engine) -> None:
|
||||
# Arrange — property A has 2 parts with distinctive areas; B has 1 with a
|
||||
# third distinctive area. If part IDs are mis-wired the floor-dimension FK
|
||||
# rows end up under the wrong property.
|
||||
base = _load_epc()
|
||||
epc_a = _with_floor_areas(base, [10.0, 20.0])
|
||||
epc_b = _with_floor_areas(base, [99.0])
|
||||
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
repo.save_batch([
|
||||
EpcSaveRequest(epc_a, property_id=2001),
|
||||
EpcSaveRequest(epc_b, property_id=2002),
|
||||
])
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
reloaded_a = repo.get_for_property(2001)
|
||||
reloaded_b = repo.get_for_property(2002)
|
||||
|
||||
# Assert — each property's building parts carry its own floor areas.
|
||||
assert reloaded_a is not None
|
||||
assert reloaded_b is not None
|
||||
areas_a = sorted(
|
||||
dim.total_floor_area_m2
|
||||
for part in reloaded_a.sap_building_parts
|
||||
for dim in part.sap_floor_dimensions
|
||||
)
|
||||
areas_b = sorted(
|
||||
dim.total_floor_area_m2
|
||||
for part in reloaded_b.sap_building_parts
|
||||
for dim in part.sap_floor_dimensions
|
||||
)
|
||||
assert areas_a == [10.0, 20.0]
|
||||
assert areas_b == [99.0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Idempotency: second save_batch() replaces, not duplicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_save_batch_is_idempotent(db_engine: Engine) -> None:
|
||||
# Arrange
|
||||
epc = _load_epc()
|
||||
requests = [EpcSaveRequest(epc, property_id=3001)]
|
||||
|
||||
with Session(db_engine) as session:
|
||||
EpcPostgresRepository(session).save_batch(requests)
|
||||
session.commit()
|
||||
|
||||
# Act — re-save the same batch.
|
||||
with Session(db_engine) as session:
|
||||
EpcPostgresRepository(session).save_batch(requests)
|
||||
session.commit()
|
||||
|
||||
# Assert — exactly one EPC survives (no duplicate rows).
|
||||
with Session(db_engine) as session:
|
||||
result = EpcPostgresRepository(session).get_for_property(3001)
|
||||
|
||||
assert result == epc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source isolation: lodged and predicted slots survive separate batch saves
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_lodged_and_predicted_batch_slots_are_independent(db_engine: Engine) -> None:
|
||||
# Arrange — two properties each get a lodged EPC and then a predicted EPC
|
||||
# via separate save_batch() calls.
|
||||
epc = _load_epc()
|
||||
property_ids = [4001, 4002]
|
||||
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
repo.save_batch([EpcSaveRequest(epc, property_id=pid, source="lodged") for pid in property_ids])
|
||||
repo.save_batch([EpcSaveRequest(epc, property_id=pid, source="predicted") for pid in property_ids])
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
repo = EpcPostgresRepository(session)
|
||||
lodged = repo.get_for_properties(property_ids)
|
||||
predicted = repo.get_predicted_for_properties(property_ids)
|
||||
|
||||
# Assert — both slots are populated for both properties.
|
||||
assert lodged == {4001: epc, 4002: epc}
|
||||
assert predicted == {4001: epc, 4002: epc}
|
||||
183
tests/repositories/plan/test_plan_batch_save.py
Normal file
183
tests/repositories/plan/test_plan_batch_save.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
"""Batch plan write path — save_batch() correctness and safety tests.
|
||||
|
||||
Guards the four user stories from #1355:
|
||||
1. save()/save_batch() parity: a single-element save_batch() produces
|
||||
identical DB state (plan row + recommendation rows) as the equivalent
|
||||
save() call.
|
||||
2. Recommendation FK isolation: two properties in the same save_batch() each
|
||||
get their own recommendation rows; no FK cross-wiring between properties.
|
||||
3. Demote correctness: a second save_batch() for the same properties demotes
|
||||
the prior default Plans and inserts fresh ones (history preserved).
|
||||
4. Non-default batch: a save_batch() where all writes have is_default=False
|
||||
leaves any pre-existing default Plan untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import Engine
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from domain.modelling.measure_type import MeasureType
|
||||
from domain.modelling.plan import Plan, PlanMeasure
|
||||
from domain.modelling.recommendation import Cost
|
||||
from domain.modelling.scoring.package_scorer import Score
|
||||
from domain.modelling.scoring.scoring import MeasureImpact
|
||||
from infrastructure.postgres.modelling import PlanModel, RecommendationModel
|
||||
from repositories.plan.plan_postgres_repository import PlanPostgresRepository
|
||||
from repositories.plan.plan_repository import PlanSaveRequest
|
||||
|
||||
|
||||
def _plan(*, sap: float = 70.0, measures: int = 1) -> Plan:
|
||||
ms: tuple[PlanMeasure, ...] = tuple(
|
||||
PlanMeasure(
|
||||
measure_type=MeasureType.CAVITY_WALL_INSULATION,
|
||||
description="Cavity wall insulation",
|
||||
cost=Cost(total=1000.0, contingency_rate=0.10),
|
||||
impact=MeasureImpact(
|
||||
sap_points=8.0,
|
||||
co2_savings_kg_per_yr=500.0,
|
||||
energy_savings_kwh_per_yr=2000.0,
|
||||
),
|
||||
kwh_savings=1500.0,
|
||||
energy_cost_savings=300.0,
|
||||
)
|
||||
for _ in range(measures)
|
||||
)
|
||||
return Plan(
|
||||
measures=ms,
|
||||
baseline=Score(sap_continuous=40.0, co2_kg_per_yr=4000.0, primary_energy_kwh_per_yr=20000.0),
|
||||
post_retrofit=Score(sap_continuous=sap, co2_kg_per_yr=3500.0, primary_energy_kwh_per_yr=18000.0),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tracer bullet: single-element save_batch() is loss-free vs save()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_single_request_save_batch_matches_save(db_engine: Engine) -> None:
|
||||
# Arrange
|
||||
plan = _plan()
|
||||
scenario_id = 7
|
||||
|
||||
with Session(db_engine) as session:
|
||||
repo = PlanPostgresRepository(session)
|
||||
save_id = repo.save(plan, property_id=5001, scenario_id=scenario_id, portfolio_id=1, is_default=True)
|
||||
batch_id = repo.save_batch([PlanSaveRequest(plan, property_id=5002, scenario_id=scenario_id, portfolio_id=1, is_default=True)])[0]
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
via_save = session.get(PlanModel, save_id)
|
||||
via_batch = session.get(PlanModel, batch_id)
|
||||
recs_save = session.exec(select(RecommendationModel).where(col(RecommendationModel.plan_id) == save_id)).all()
|
||||
recs_batch = session.exec(select(RecommendationModel).where(col(RecommendationModel.plan_id) == batch_id)).all()
|
||||
|
||||
# Assert — both paths produce one plan row + one recommendation row with the
|
||||
# same field values (modulo property_id which differs by design).
|
||||
assert via_save is not None
|
||||
assert via_batch is not None
|
||||
assert via_save.is_default is True
|
||||
assert via_batch.is_default is True
|
||||
assert via_save.post_sap_points == via_batch.post_sap_points
|
||||
assert via_save.post_co2_emissions == via_batch.post_co2_emissions
|
||||
assert via_save.co2_savings == via_batch.co2_savings
|
||||
assert len(recs_save) == 1
|
||||
assert len(recs_batch) == 1
|
||||
assert recs_save[0].type == recs_batch[0].type
|
||||
assert recs_save[0].estimated_cost == recs_batch[0].estimated_cost
|
||||
assert recs_save[0].sap_points == recs_batch[0].sap_points
|
||||
assert recs_batch[0].plan_id == batch_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FK isolation: recommendation rows must not be crossed between properties
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_multi_property_recommendation_fks_are_not_crossed(db_engine: Engine) -> None:
|
||||
# Arrange — property A gets 2 measures, property B gets 1 measure.
|
||||
plan_a = _plan(measures=2)
|
||||
plan_b = _plan(measures=1)
|
||||
|
||||
with Session(db_engine) as session:
|
||||
[id_a, id_b] = PlanPostgresRepository(session).save_batch([
|
||||
PlanSaveRequest(plan_a, property_id=6001, scenario_id=7, portfolio_id=1, is_default=True),
|
||||
PlanSaveRequest(plan_b, property_id=6002, scenario_id=7, portfolio_id=1, is_default=True),
|
||||
])
|
||||
session.commit()
|
||||
|
||||
# Act
|
||||
with Session(db_engine) as session:
|
||||
recs_a = session.exec(select(RecommendationModel).where(col(RecommendationModel.plan_id) == id_a)).all()
|
||||
recs_b = session.exec(select(RecommendationModel).where(col(RecommendationModel.plan_id) == id_b)).all()
|
||||
|
||||
# Assert — A has 2 rows, B has 1; none cross-wired.
|
||||
assert len(recs_a) == 2
|
||||
assert len(recs_b) == 1
|
||||
assert all(r.plan_id == id_a and r.property_id == 6001 for r in recs_a)
|
||||
assert all(r.plan_id == id_b and r.property_id == 6002 for r in recs_b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Demote correctness: second save_batch() demotes prior defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_second_save_batch_demotes_prior_default_plans(db_engine: Engine) -> None:
|
||||
# Arrange — first batch creates default Plans for two properties.
|
||||
plan = _plan()
|
||||
requests = [
|
||||
PlanSaveRequest(plan, property_id=7001, scenario_id=7, portfolio_id=1, is_default=True),
|
||||
PlanSaveRequest(plan, property_id=7002, scenario_id=7, portfolio_id=1, is_default=True),
|
||||
]
|
||||
with Session(db_engine) as session:
|
||||
first_ids = PlanPostgresRepository(session).save_batch(requests)
|
||||
session.commit()
|
||||
|
||||
# Act — re-run the same batch; new Plans should become default, old ones demoted.
|
||||
with Session(db_engine) as session:
|
||||
second_ids = PlanPostgresRepository(session).save_batch(requests)
|
||||
session.commit()
|
||||
|
||||
# Assert — history is preserved (4 plan rows total); exactly one default per property.
|
||||
with Session(db_engine) as session:
|
||||
rows_7001 = session.exec(select(PlanModel).where(col(PlanModel.property_id) == 7001)).all()
|
||||
rows_7002 = session.exec(select(PlanModel).where(col(PlanModel.property_id) == 7002)).all()
|
||||
|
||||
by_id_7001 = {p.id: p for p in rows_7001}
|
||||
by_id_7002 = {p.id: p for p in rows_7002}
|
||||
|
||||
assert len(rows_7001) == 2
|
||||
assert len(rows_7002) == 2
|
||||
assert by_id_7001[first_ids[0]].is_default is False
|
||||
assert by_id_7001[second_ids[0]].is_default is True
|
||||
assert by_id_7002[first_ids[1]].is_default is False
|
||||
assert by_id_7002[second_ids[1]].is_default is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-default batch: existing default Plan is untouched
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_non_default_save_batch_does_not_demote_existing_default(db_engine: Engine) -> None:
|
||||
# Arrange — a default Plan already exists for the property.
|
||||
plan = _plan()
|
||||
with Session(db_engine) as session:
|
||||
default_id = PlanPostgresRepository(session).save(
|
||||
plan, property_id=8001, scenario_id=7, portfolio_id=1, is_default=True
|
||||
)
|
||||
session.commit()
|
||||
|
||||
# Act — save a non-default Plan via save_batch(); no demote UPDATE should fire.
|
||||
with Session(db_engine) as session:
|
||||
PlanPostgresRepository(session).save_batch([
|
||||
PlanSaveRequest(plan, property_id=8001, scenario_id=7, portfolio_id=1, is_default=False),
|
||||
])
|
||||
session.commit()
|
||||
|
||||
# Assert — the original default Plan is still the default.
|
||||
with Session(db_engine) as session:
|
||||
rows = session.exec(select(PlanModel).where(col(PlanModel.property_id) == 8001)).all()
|
||||
by_id = {p.id: p for p in rows}
|
||||
|
||||
assert len(rows) == 2
|
||||
assert by_id[default_id].is_default is True
|
||||
assert sum(1 for p in rows if p.is_default) == 1
|
||||
Loading…
Add table
Reference in a new issue