Merge branch 'main' into community-heating-hhrsh

This commit is contained in:
Daniel Roth 2026-07-01 13:58:29 +00:00
commit 355a632e5f
11 changed files with 443 additions and 6 deletions

View file

@ -10,6 +10,7 @@ from applications.landlord_description_overrides.landlord_description_overrides_
from domain.epc.property_overrides.built_form_type import BuiltFormType
from domain.epc.property_overrides.construction_age_band import ConstructionAgeBand
from domain.epc.property_overrides.glazing_type import GlazingType
from domain.epc.property_overrides.glazing_mix_guard import glazing_mix_guard
from domain.epc.property_overrides.main_fuel_type import MainFuelType
from domain.epc.property_overrides.main_heating_system_type import MainHeatingSystemType
from domain.epc.property_overrides.property_type import PropertyType
@ -146,8 +147,15 @@ def _build_columns(
"glazing": lambda src: ClassifiableColumn(
name="glazing",
source_column=src,
classifier=ChatGptColumnClassifier(
chat_gpt, GlazingType, GlazingType.UNKNOWN
# An aggregate glazing mix ("40% double, 60% single") can't be applied
# per-window, so the deterministic guard resolves the structured split
# to MIXED (no overlay → keep the cert's per-window glazing) and the LLM
# handles uniform / varied phrasings (#1376, ADR-0042).
classifier=GuardedColumnClassifier(
guard=glazing_mix_guard,
fallback=ChatGptColumnClassifier(
chat_gpt, GlazingType, GlazingType.UNKNOWN
),
),
repo=LandlordOverridesRepository[GlazingType](
session, LandlordGlazingOverrideRow

View file

@ -616,15 +616,35 @@ class EpcPropertyDataMapper:
insulated_door_count=schema.insulated_door_count,
draughtproofed_door_count=None,
percent_draughtproofed=schema.percent_draughtproofed,
# ADR-0028: the schema lodges total + low-energy OUTLET counts, not a
# bulb split — mirror the 17.1/18.0/20.0 paths (low-energy outlets →
# low_energy bulbs; the remainder → incandescent). Hardcoding all
# bulb counts to 0 dropped the lodged lighting mix, understating the
# low-energy lighting credit in the SAP lighting-energy calc.
led_fixed_lighting_bulbs_count=0,
cfl_fixed_lighting_bulbs_count=0,
incandescent_fixed_lighting_bulbs_count=0,
incandescent_fixed_lighting_bulbs_count=(
schema.fixed_lighting_outlets_count
- schema.low_energy_fixed_lighting_outlets_count
),
low_energy_fixed_lighting_bulbs_count=schema.low_energy_fixed_lighting_outlets_count,
roofs=EpcPropertyDataMapper._map_energy_elements(schema.roofs),
walls=EpcPropertyDataMapper._map_energy_elements(schema.walls),
floors=EpcPropertyDataMapper._map_energy_elements(schema.floors),
main_heating=EpcPropertyDataMapper._map_energy_elements(
schema.main_heating
),
# First control system if multiple — mirrors the 19.0/21.0.1 paths.
# 17.0 lodges main_heating_controls (List[EnergyElement]) but omitted
# it, leaving main_heating_controls=None so the heating-control
# recommendations (programmer/room-stat/TRVs) had nothing to read.
main_heating_controls=(
EpcPropertyDataMapper._map_energy_element(
schema.main_heating_controls[0]
)
if schema.main_heating_controls
else None
),
window=EpcPropertyDataMapper._map_energy_element(schema.window),
lighting=EpcPropertyDataMapper._map_energy_element(schema.lighting),
hot_water=EpcPropertyDataMapper._map_energy_element(schema.hot_water),
@ -1599,9 +1619,18 @@ class EpcPropertyDataMapper:
insulated_door_count=schema.insulated_door_count,
draughtproofed_door_count=None,
percent_draughtproofed=schema.percent_draughtproofed,
# ADR-0028: the schema lodges total + low-energy OUTLET counts, not a
# bulb split — mirror the 17.1/18.0/20.0 paths (low-energy outlets →
# low_energy bulbs; the remainder → incandescent). Hardcoding all
# bulb counts to 0 dropped the lodged lighting mix, understating the
# low-energy lighting credit in the SAP lighting-energy calc.
led_fixed_lighting_bulbs_count=0,
cfl_fixed_lighting_bulbs_count=0,
incandescent_fixed_lighting_bulbs_count=0,
incandescent_fixed_lighting_bulbs_count=(
schema.fixed_lighting_outlets_count
- schema.low_energy_fixed_lighting_outlets_count
),
low_energy_fixed_lighting_bulbs_count=schema.low_energy_fixed_lighting_outlets_count,
roofs=EpcPropertyDataMapper._map_energy_elements(schema.roofs),
walls=EpcPropertyDataMapper._map_energy_elements(schema.walls),
floors=EpcPropertyDataMapper._map_energy_elements(schema.floors),

View file

@ -29,3 +29,14 @@ The landlord glazing override **reconciles against the cert's per-window composi
- Genuine landlord-vs-cert glazing disagreements are **surfaced** rather than silently trusted (old behaviour: flatten to double) or silently overwritten.
- **Alternatives rejected.** *Dominant type wins* — still clobbers per-window variation. *Carry a full landlord composition to re-derive per-window types* — per-window assignment from an aggregate is unknowable; we decline to fabricate it. *Deterministic regex of "X% double, Y% single"* — brittle against varied input, which is the LLM's job.
- Pairs with the per-window fidelity the calculator already relies on: the reconciliation is only correct on a faithful Effective EPC whose `sap_windows` round-trip (cf. ADR-0040).
## Amendment (#1376 implementation): the reconcile collapses to a `MIXED` sentinel
Implementing this surfaced that the full "carry a proportion → compare to the cert → flag when far" design buys less than its cost. The cert **already** carries the best-possible per-window mapping (`sap_windows`, each with its own `glazing_type`); a landlord aggregate ("40% double, 60% single") says *how much* is double but **not which windows**, so applying it would overwrite real per-window data with an arbitrary guess. Per-window assignment from an aggregate is therefore not just unknowable but strictly worse than the cert — so a mixed override **always defers**, regardless of the exact proportion. The proportion comparison only ever fed the "flag when far" branch, and carrying a proportion needs a new schema field (an FE/Drizzle dependency).
**Realized decision.** Add a `GlazingType.MIXED` sentinel. The LLM classifies a genuine aggregate-mix to `MIXED` (the LLM handles the varied phrasings — no regex); `MIXED` is absent from the overlay's `_GLAZING_CODES`, so it resolves to **no overlay** and `_fold_glazing` never runs — the cert's per-window glazing is preserved. A **uniform** assertion (one type ≳ 90%) still resolves to that type and is applied to every window (unambiguous — no guess). Existing mixed-as-double rows are reclassified to `MIXED` (a deterministic parse of the `%X / %Y` format is acceptable for fixing *known* rows). The "uniform vs mixed" threshold (~90% one type) is tunable, pinned against the real distribution.
**Consequences of the amendment.**
- The defer behaviour needs **no apply-seam logic and no proportion field** — it falls out of "no overlay". The slice is: add the `MIXED` member, give the classifier the option, reclassify the ~319 rows.
- `MIXED` is a **new FE-owned pgEnum value** (one member) — added by the FE owner (Dan); `property_overrides` (TEXT) reclassify runs immediately, the cache pgEnum write defers until the migration (the Class-A/B pattern).
- The **"flag when landlord disagrees with the cert" is dropped for now** — since a mix always defers, the flag was informational only. `MIXED` is the hook if a future slice wants to surface those disagreements.

View file

@ -0,0 +1,40 @@
from __future__ import annotations
import re
from typing import Optional
from domain.epc.property_overrides.glazing_type import GlazingType
# "N% <base glazing type>" — the era (2002 / pre-2002) is irrelevant to whether it
# is a mix, so only the base type is captured.
_PERCENT_TYPE = re.compile(r"(\d+)%\s*(single|double|triple|secondary)")
# One base type at or above this share is a uniform assertion, not a mix — applied
# as that type (a near-uniform reglaze), not deferred.
_UNIFORM_THRESHOLD_PERCENT = 90
def glazing_mix_guard(description: str) -> Optional[GlazingType]:
"""Deterministically resolve an aggregate-mix glazing description to ``MIXED``.
A landlord glazing description is often a whole-dwelling proportion split
("40% double glazing 2002 or later, 60% single glazing"). Such a mix cannot be
faithfully applied per-window it says *how much* is each type, not *which*
windows and the cert already carries the true per-window glazing, so a mix
must resolve to ``GlazingType.MIXED`` (no overlay the cert is kept), never to
a single dominant type that would flatten every window (ADR-0042).
Recognises the structured ``"N% <type>, M% <type>"`` split and returns
``MIXED`` when it is a genuine mix two or more glazing types present and none
dominant ( the uniform threshold). Returns ``None`` for a uniform / near-
uniform assertion or an unparseable description, so the LLM classifier still
resolves it (a uniform reglaze is applied; varied phrasings are the LLM's job).
"""
matches = _PERCENT_TYPE.findall(description.lower())
base_types = {base_type for _, base_type in matches}
if len(base_types) < 2:
# One (or no) glazing type named — a uniform assertion, not a mix.
return None
if max(int(percent) for percent, _ in matches) >= _UNIFORM_THRESHOLD_PERCENT:
# One type dominates — treat as that (near-)uniform type, not a mix.
return None
return GlazingType.MIXED

View file

@ -21,4 +21,11 @@ class GlazingType(Enum):
DOUBLE_PRE_2002 = "Double glazing, pre-2002"
TRIPLE_PRE_2002 = "Triple glazing, pre-2002"
TRIPLE_POST_2002 = "Triple glazing, 2002 or later"
# An aggregate mix of glazing types across the dwelling ("40% double, 60%
# single"). Deliberately absent from the overlay's `_GLAZING_CODES`, so it
# resolves to no overlay and the cert's per-window `sap_windows` are kept
# rather than flattened to one type — a whole-dwelling proportion cannot say
# which windows are which (ADR-0042 amendment, #1376). The FE-owned pgEnum
# gains this member via a Drizzle migration.
MIXED = "Mixed glazing"
UNKNOWN = "Unknown"

View file

@ -0,0 +1,147 @@
"""One-time re-classification of aggregate-mix glazing overrides flattened to Double.
#1376 / ADR-0042: a landlord glazing description that is an aggregate mix
("40% double glazing 2002 or later, 60% single glazing") was classified to a single
``"Double glazing…"`` value whenever any double was present, and the overlay then
overwrote **every** window to double over-crediting a predominantly-single dwelling
and clobbering the cert's per-window glazing. A whole-dwelling proportion cannot say
*which* windows are which, so a mix must resolve to ``GlazingType.MIXED`` (no overlay
the cert's per-window ``sap_windows`` are kept).
The live classifier now applies ``glazing_mix_guard`` deterministically (so new
intakes of the structured split are correct); this fixes the rows written before it.
The **same guard** decides the correction here, so the backfill and the live path
cannot drift.
Updates the TEXT ``property_overrides.override_value`` (what the modelling reads
the immediate fix). The ``landlord_glazing_overrides.value`` classifier cache is a
``glazing`` **pgEnum**; ``MIXED`` is a new FE-owned value, so cache writes are
**deferred** until the Drizzle migration adds it (the Class-A/B pattern).
DRY-RUN BY DEFAULT: prints the counts it would change and writes nothing. Pass
``--apply`` to execute inside a transaction. Idempotent only rows whose stored
value differs from the target member are touched.
"""
from __future__ import annotations
import argparse
from collections.abc import Iterable
from sqlalchemy import Connection, text
from domain.epc.property_overrides.glazing_mix_guard import glazing_mix_guard
from scripts.e2e_common import build_engine, load_env
def mixed_glazing_corrections(
stored: Iterable[tuple[str, str]],
) -> dict[str, str]:
"""``(description, stored override_value)`` → the ``MIXED`` value, for the
descriptions the glazing mix guard resolves to a mix whose stored value is not
already ``MIXED``. Uniform / unparseable descriptions and rows already on
``MIXED`` are omitted, so re-running against corrected data is a no-op."""
corrections: dict[str, str] = {}
for description, value in stored:
member = glazing_mix_guard(description)
if member is not None and value != member.value:
corrections[description] = member.value
return corrections
_DISTINCT = text(
"""
SELECT DISTINCT lower(original_spreadsheet_description) AS description,
override_value AS value
FROM property_overrides
WHERE override_component = 'glazing'
"""
)
_OVERRIDES_UPDATE = text(
"""
UPDATE property_overrides
SET override_value = :new_value
WHERE override_component = 'glazing'
AND lower(original_spreadsheet_description) = :description
AND override_value <> :new_value
"""
)
_OVERRIDES_COUNT = text(
"""
SELECT count(*) FROM property_overrides
WHERE override_component = 'glazing'
AND lower(original_spreadsheet_description) = :description
AND override_value <> :new_value
"""
)
_CACHE_UPDATE = text(
"""
UPDATE landlord_glazing_overrides
SET value = :new_value, updated_at = now()
WHERE lower(description) = :description
AND value::text <> :new_value
"""
)
_ENUM_VALUES = text(
"SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid "
"WHERE t.typname = 'glazing'"
)
def reclassify(conn: Connection, *, apply: bool) -> tuple[int, set[str]]:
"""Re-map aggregate-mix glazing overrides onto MIXED. Returns the number of
``property_overrides`` rows found and the set of target values the live
``glazing`` pgEnum does not yet carry (cache-deferred until the FE migration)."""
stored = [(r.description, r.value) for r in conn.execute(_DISTINCT)]
enum_values = {r[0] for r in conn.execute(_ENUM_VALUES)}
total = 0
deferred: set[str] = set()
for description, new_value in mixed_glazing_corrections(stored).items():
params = {"description": description, "new_value": new_value}
total += conn.execute(_OVERRIDES_COUNT, params).scalar() or 0
in_enum = new_value in enum_values
if not in_enum:
deferred.add(new_value)
if apply:
conn.execute(_OVERRIDES_UPDATE, params)
if in_enum:
conn.execute(_CACHE_UPDATE, params)
return total, deferred
def main() -> None:
load_env()
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--apply",
action="store_true",
help="execute the updates (default: dry-run, writes nothing)",
)
args = parser.parse_args()
engine = build_engine()
with engine.begin() as conn:
conn.execute(text("SET statement_timeout = 120000"))
total, deferred = reclassify(conn, apply=args.apply)
verb = "re-classified" if args.apply else "would re-classify"
print(
f"{verb} {total} aggregate-mix glazing override row(s) from a flattened "
"Double onto MIXED (property_overrides / TEXT); the cert's per-window "
"glazing is kept."
)
if deferred:
print(
f"\n{len(deferred)} target value(s) NOT yet in the glazing pgEnum — "
"their classifier-cache rows are deferred until the FE-repo enum "
"migration adds these members (property_overrides was still updated, "
"which is what the modelling reads):"
)
for value in sorted(deferred):
print(f" {value!r}")
if not args.apply:
print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,57 @@
"""Regression: the degraded RdSAP paths (17.0, 19.0) must carry the lodged
fixed-lighting outlet counts, like the reference-complete 17.1/18.0/20.0 paths.
PRD #1385 (mapper normalization), lighting family. Both paths hardcoded every
bulb count to 0, dropping the lodged lighting mix. The schema lodges total +
low-energy OUTLET counts (ADR-0028); the reference paths split them as
low-energy low_energy bulbs and the remainder incandescent, feeding the SAP
lighting-energy calc. (21.0.x use a different bulb-based lighting schema and are
out of scope for this slice.)
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
# 17.0 cert: fixed_lighting_outlets_count=10, low_energy=5 → incandescent 5.
_FIXTURE_17_0 = Path("tests/fixtures/epc_prediction/PE71NT/cert-26619d8e7f8e.json")
# 19.0 cert: fixed_lighting_outlets_count=8, low_energy=5 → incandescent 3.
_FIXTURE_19_0 = Path("tests/fixtures/epc_prediction/PE71NT/cert-21b634bc4b59.json")
def _load(path: Path, schema_type: str) -> dict[str, Any]:
data: dict[str, Any] = json.loads(path.read_text())
assert data["schema_type"] == schema_type
return data
def test_17_0_path_carries_lighting_outlet_counts() -> None:
data = _load(_FIXTURE_17_0, "RdSAP-Schema-17.0")
assert data["fixed_lighting_outlets_count"] == 10
assert data["low_energy_fixed_lighting_outlets_count"] == 5
epc = EpcPropertyDataMapper.from_api_response(data)
# Was hardcoded 0/0/0. Low-energy outlets carry through; the remainder is
# incandescent; led/cfl stay 0 (reduced-field certs don't split those).
assert epc.low_energy_fixed_lighting_bulbs_count == 5
assert epc.incandescent_fixed_lighting_bulbs_count == 5
assert epc.led_fixed_lighting_bulbs_count == 0
assert epc.cfl_fixed_lighting_bulbs_count == 0
def test_19_0_path_carries_lighting_outlet_counts() -> None:
data = _load(_FIXTURE_19_0, "RdSAP-Schema-19.0")
assert data["fixed_lighting_outlets_count"] == 8
assert data["low_energy_fixed_lighting_outlets_count"] == 5
epc = EpcPropertyDataMapper.from_api_response(data)
assert epc.low_energy_fixed_lighting_bulbs_count == 5
assert epc.incandescent_fixed_lighting_bulbs_count == 3
assert epc.led_fixed_lighting_bulbs_count == 0
assert epc.cfl_fixed_lighting_bulbs_count == 0

View file

@ -0,0 +1,48 @@
"""Regression: the RdSAP-Schema-17.0 mapper path must carry the lodged
main_heating_controls, like every other API path (19.0/20.0/21.x).
PRD #1385 (mapper normalization). 17.0 lodges `main_heating_controls`
(List[EnergyElement]) but the mapper omitted it, leaving the domain field None
while the 19.0/21.0.1 paths map the first control via `_map_energy_element`. The
mapped element is the same EnergyElement type 17.0 already carries for
`main_heating` / `window`, so this is straight parity.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
# 17.0 cert lodging a main-heating control ("Programmer, TRVs and bypass").
_FIXTURE = Path("tests/fixtures/epc_prediction/PE71NT/cert-26619d8e7f8e.json")
def _load() -> dict[str, Any]:
data: dict[str, Any] = json.loads(_FIXTURE.read_text())
assert data["schema_type"] == "RdSAP-Schema-17.0"
return data
def test_17_0_maps_lodged_main_heating_controls() -> None:
data = _load()
assert data["main_heating_controls"], "fixture must lodge a control"
epc = EpcPropertyDataMapper.from_api_response(data)
# Was None on the 17.0 path; now carries the first lodged control element.
assert epc.main_heating_controls is not None
assert epc.main_heating_controls.description == "Programmer, TRVs and bypass"
assert epc.main_heating_controls.energy_efficiency_rating == 3
def test_17_0_no_controls_maps_to_none() -> None:
# An empty control list maps to None (no phantom element), like the siblings.
data = _load()
data["main_heating_controls"] = []
epc = EpcPropertyDataMapper.from_api_response(data)
assert epc.main_heating_controls is None

View file

@ -0,0 +1,39 @@
from __future__ import annotations
import pytest
from domain.epc.property_overrides.glazing_type import GlazingType
from domain.epc.property_overrides.glazing_mix_guard import glazing_mix_guard
def test_a_genuine_percentage_mix_classifies_as_mixed() -> None:
# Arrange — the bug: an aggregate mix the LLM would flatten to "Double".
description = "40% double glazing 2002 or later, 60% single glazing"
# Act
result = glazing_mix_guard(description)
# Assert — a mix defers to the cert's per-window glazing, not a whole-dwelling
# type.
assert result is GlazingType.MIXED
@pytest.mark.parametrize(
"description",
[
"100% double glazing 2002 or later", # uniform — one type
"96% double glazing 2002 or later, 4% single glazing", # >= 90% -> uniform
"80% double glazing 2002 or later, 20% double glazing pre-2002", # same base type
"some double glazing and a bit of single", # unparseable — no percentages
"",
],
)
def test_uniform_or_unparseable_glazing_defers_to_the_llm(description: str) -> None:
# The guard only claims a genuine percentage mix; a uniform assertion (applied
# as that type) or a varied phrasing is left for the LLM classifier.
# Act
result = glazing_mix_guard(description)
# Assert
assert result is None

View file

@ -57,6 +57,18 @@ def test_unresolvable_glazing_produces_no_overlay(glazing_value: str) -> None:
assert simulation is None
def test_a_glazing_mix_produces_no_overlay_so_the_cert_is_kept() -> None:
# A MIXED aggregate resolves to no overlay: a whole-dwelling proportion cannot
# say which windows are which, so the cert's per-window `sap_windows` are kept
# rather than flattened to one type (ADR-0042 amendment, #1376).
# Act
simulation = glazing_overlay_for(GlazingType.MIXED.value, 0)
# Assert
assert simulation is None
def test_glazing_override_remaps_every_window_and_clears_lodged_u() -> None:
# Arrange — baseline windows are double glazed (code 2, lodged U 2.8); the
# landlord corrects the whole dwelling to single glazing.
@ -74,14 +86,18 @@ def test_glazing_override_remaps_every_window_and_clears_lodged_u() -> None:
assert all(w.window_transmission_details is None for w in result.sap_windows)
_NO_OVERLAY_SENTINELS = {GlazingType.UNKNOWN, GlazingType.MIXED}
@pytest.mark.parametrize(
"member", [m for m in GlazingType if m is not GlazingType.UNKNOWN]
"member", [m for m in GlazingType if m not in _NO_OVERLAY_SENTINELS]
)
def test_every_resolvable_glazing_value_decodes_to_a_code(
member: GlazingType,
) -> None:
# A classifier emits a GlazingType value; if the overlay can't decode it the
# override silently no-ops. Every non-UNKNOWN member must resolve.
# override silently no-ops. Every member except the deliberate no-overlay
# sentinels (UNKNOWN, MIXED) must resolve to a code.
# Act
simulation = glazing_overlay_for(member.value, 0)

View file

@ -0,0 +1,35 @@
"""The mixed-glazing reclassify maps a flattened aggregate mix onto MIXED, leaves
uniform / already-correct rows alone, and is idempotent (#1376 / ADR-0042)."""
from __future__ import annotations
from scripts.reclassify_mixed_glazing import mixed_glazing_corrections
def test_flattened_mix_rows_are_corrected_to_mixed() -> None:
# Arrange — an aggregate mix flattened to Double (the bug), a near-uniform mix
# and a clean uniform row (both correctly Double), and a pure single row.
stored = [
("40% double glazing 2002 or later, 60% single glazing", "Double glazing, 2002 or later"),
("96% double glazing 2002 or later, 4% single glazing", "Double glazing, 2002 or later"),
("100% double glazing 2002 or later", "Double glazing, 2002 or later"),
("100% single glazing", "Single glazing"),
]
# Act
corrections = mixed_glazing_corrections(stored)
# Assert — only the genuine mix is re-mapped; near-uniform and uniform stay.
assert corrections == {
"40% double glazing 2002 or later, 60% single glazing": "Mixed glazing"
}
def test_already_mixed_rows_need_no_change() -> None:
# Act / Assert — idempotent.
assert (
mixed_glazing_corrections(
[("40% double glazing 2002 or later, 60% single glazing", "Mixed glazing")]
)
== {}
)