diff --git a/applications/landlord_description_overrides/handler.py b/applications/landlord_description_overrides/handler.py index bc5fe8ee9..6d87490e9 100644 --- a/applications/landlord_description_overrides/handler.py +++ b/applications/landlord_description_overrides/handler.py @@ -13,6 +13,7 @@ from domain.epc.property_overrides.glazing_type import GlazingType from domain.epc.property_overrides.glazing_mix_guard import glazing_mix_guard from domain.epc.property_overrides.main_fuel_type import MainFuelType from domain.epc.property_overrides.main_heating_system_type import MainHeatingSystemType +from domain.epc.property_overrides.main_heating_guard import main_heating_guard from domain.epc.property_overrides.property_type import PropertyType from domain.epc.property_overrides.roof_type import RoofType from domain.epc.property_overrides.roof_party_ceiling_guard import ( @@ -193,8 +194,15 @@ def _build_columns( "main_heating_system": lambda src: ClassifiableColumn( name="main_heating_system", source_column=src, - classifier=ChatGptColumnClassifier( - chat_gpt, MainHeatingSystemType, MainHeatingSystemType.UNKNOWN + # High heat retention storage heaters have no LLM target distinct from + # old storage, so they were funnelled into "Electric storage heaters, + # old" (401); the deterministic guard resolves HHRSH to its own + # archetype (SAP 409) and the LLM handles the rest (#1376, ADR-0044). + classifier=GuardedColumnClassifier( + guard=main_heating_guard, + fallback=ChatGptColumnClassifier( + chat_gpt, MainHeatingSystemType, MainHeatingSystemType.UNKNOWN + ), ), repo=LandlordOverridesRepository[MainHeatingSystemType]( session, LandlordMainHeatingSystemOverrideRow diff --git a/docs/adr/0044-hhrsh-baseline-archetype-drags-its-intrinsic-controls.md b/docs/adr/0044-hhrsh-baseline-archetype-drags-its-intrinsic-controls.md new file mode 100644 index 000000000..354844e20 --- /dev/null +++ b/docs/adr/0044-hhrsh-baseline-archetype-drags-its-intrinsic-controls.md @@ -0,0 +1,94 @@ +# A baseline HHRSH override drags its intrinsic SAP-409 controls, not the generic manual storage default + +## Status + +accepted + +## Context + +A **Landlord Override** naming *"High heat retention storage heaters"* (HHRSH) had +no home in the heating taxonomy: `MainHeatingSystemType` carried +`ELECTRIC_STORAGE_OLD` (SAP 401), `_SLIMLINE` (402), `_CONVECTOR` (403) and +`_FAN` (404), but no High-Heat-Retention member. So the LLM classifier funnelled +the **best modern storage** into **`"Electric storage heaters, old"` (401)** — the +**worst** large-volume type. An audit found **111** `property_overrides` rows on +that mapping (`'Electric Storage Systems: High heat retention storage heaters'` → +`'Electric storage heaters, old'`). Same missing-archetype defect as ADR-0041. + +HHRSH is a distinct, modellable RdSAP system: **SAP Table 4a code 409** (high heat +retention storage heaters), which the calculator already scores +(`sap_efficiencies` carries `409: 1.00`). The **HHRSH *measure*** already installs +this exact code (`_HHR_STORAGE_SAP_CODE = 409`, +`domain/modelling/generators/heating_recommendation.py`) on a **Dual** off-peak +meter with **control 2404**. + +The subtlety is the **charge control**. `main_heating_system_overlay._control_for` +drags the **conservative manual** charge control (`2401`) for *every* storage code +401–409, following ADR-0041's amendment ("an *unobserved* storage charge control +defaults to the conservative manual one"). Table 4e Group 4's mean-internal- +temperature adjustments are **manual +0.7 °C, automatic/Celect +0.4 °C, HHR 0 °C** +— so a manual default carries a penalty. Applying it to a 409 heater would swap +"scored as old-storage 401" for "scored as 409-but-penalised" — still under- +crediting the 111 dwellings, and pairing 409 with manual control is not a +combination RdSAP produces. + +**HHRSH has a single, intrinsic charge control (2404).** The control is therefore +*named by the archetype*, not *unobserved* — unlike a generic storage heater whose +control genuinely is unknown. So ADR-0041's manual-default rule does not apply to +409. + +## Decision + +Add the baseline HHRSH archetype and let it drag its **coherent, intrinsic** +companions — mirroring how a gas-boiler archetype drags full modern controls +(`_control_for`) rather than a conservative default. + +1. **`MainHeatingSystemType.ELECTRIC_STORAGE_HIGH_HEAT_RETENTION` = + `"Electric storage heaters, high heat retention"`**, mapped in + `_MAIN_HEATING_CODES` to **SAP Table 4a code 409**. +2. **Control 2404 (HHRSH controls), not the manual default.** `_control_for` + special-cases 409 → `2404` ahead of the generic storage → `2401`. HHRSH's single + intrinsic control is archetype-implied, an **exception to ADR-0041's + unobserved-storage manual default** (which stands for 401–404). +3. **Electricity fuel (RdSAP 29).** `_natural_fuel_for` drags electricity for 409, + per ADR-0041's "storage → electricity" — so a mis-lodged non-electric fuel can't + leave an electric-storage dwelling incoherent. Scoped to 409 ("scoped per family + as archetypes land"); the pre-existing 401–404 fuel-drag gap is not touched here. +4. **Dual/off-peak meter falls out of the code** — 409 is already in + `_ASSUMED_DUAL_METER_CODES`, so the off-peak meter drags automatically (ADR-0035 + coherent companions). +5. **Deterministic guard + reclassify, TEXT-first / pgEnum-deferred.** A + `main_heating_guard` recognises the structured `"Electric Storage Systems: High + heat retention storage heaters"` phrasing (wired via `GuardedColumnClassifier`); + `scripts/reclassify_hhrsh_storage.py` reuses the same guard as its decision + function, updates `property_overrides.override_value` (TEXT) unconditionally, and + defers the `landlord_main_heating_system_overrides.value` pgEnum-cache write for + the new value. + +## Consequences + +- The 111 HHRSH dwellings score as **409 + HHR control 2404 + off-peak meter + + electricity** — the efficient modern archetype — instead of the worst old + large-volume 401. Correct Rebaselining (ADR-0039). +- **FE-owned pgEnum addition (Dan):** one new `main_heating_system` value, + `"Electric storage heaters, high heat retention"` (memory + `main-heating-system-pgenum-is-fe-owned`). TEXT read path is fixed on merge; the + cache write defers until the migration lands. +- **Baseline vs measure stay distinct but consistent:** both use code 409 + + control 2404; the measure additionally installs a designed HW cylinder / immersion + end-state, which a baseline (existing-system) override does not. +- Extends [ADR-0041](0041-landlord-heating-classification-targets-a-complete-modellable-taxonomy.md) + (complete taxonomy; **refines** its manual-default amendment for the single- + control HHRSH case) and [ADR-0035](0035-coherent-heating-system-synthesis.md) + (companions drag from the code). + +### Alternatives rejected + +- **409 + manual control (2401), holding to ADR-0041's default.** Rejected: HHRSH's + control is intrinsic (single option), not unobserved; the +0.7 °C manual penalty + under-credits, and 409+manual is not a real RdSAP combination. +- **Reuse an existing storage archetype (401–404).** Rejected: none carries the + HHR retention/control — the whole point of the mis-score. +- **Leave fuel unresolved (as 401–404 do).** Rejected for 409: an explicit + electricity drag keeps the system coherent if the lodged fuel is wrong; the older + archetypes' gap is a separate, out-of-scope cleanup. diff --git a/domain/epc/property_overlays/main_heating_system_overlay.py b/domain/epc/property_overlays/main_heating_system_overlay.py index d14f9e79c..7d34546ce 100644 --- a/domain/epc/property_overlays/main_heating_system_overlay.py +++ b/domain/epc/property_overlays/main_heating_system_overlay.py @@ -67,6 +67,12 @@ _ASSUMED_DUAL_METER_CODES = OFF_PEAK_IMPLYING_HEATING_CODES | _ROOM_HEATER_CODES # systems that take a charge control. _MANUAL_CHARGE_CONTROL = 2401 _STORAGE_HEATER_CODES = frozenset(range(401, 410)) +# High heat retention storage heaters (SAP Table 4a 409) take a SINGLE intrinsic +# charge control — Table 4e 2404 (HHR, 0 C adjustment). It is named by the +# archetype, not unobserved, so 409 drags 2404 rather than the manual default +# (ADR-0044) — the only storage code that overrides `_MANUAL_CHARGE_CONTROL`. +_HHR_STORAGE_CODE = 409 +_HHR_CHARGE_CONTROL = 2404 # SAP Table 4a category 10 ("Room heaters") and its conservative Table 4e Group 6 # control. A landlord names a room heater, not its control; code 2601 ("no @@ -162,6 +168,10 @@ _MAIN_HEATING_CODES: dict[str, int] = { "Electric storage heaters, slimline": 402, "Electric storage heaters, convector": 403, "Electric storage heaters, fan": 404, + # High heat retention storage heaters — SAP Table 4a 409. Its intrinsic HHR + # charge control (2404) and electricity fuel drag from the code below, not the + # generic manual-storage default (ADR-0044). + "Electric storage heaters, high heat retention": 409, "Direct-acting electric": 191, "Electric room heaters": 691, "Solid fuel room heater, closed": 633, @@ -203,6 +213,8 @@ def _control_for(code: int) -> Optional[int]: conservative manual charge control for storage heaters, full modern controls for a gas boiler, None for systems that take neither (direct-acting electric). Overwrites a stale control inherited from the system being replaced.""" + if code == _HHR_STORAGE_CODE: + return _HHR_CHARGE_CONTROL if code in _STORAGE_HEATER_CODES: return _MANUAL_CHARGE_CONTROL if code in _GAS_BOILER_CODES: @@ -229,7 +241,11 @@ def _natural_fuel_for(code: int) -> Optional[int]: """The fuel an archetype unambiguously implies (a coherent default), or None where the fuel is ambiguous and must come from the `main_fuel` override. A present `main_fuel` override wins by last-wins composition.""" - if code in _ELECTRIC_ROOM_HEATER_CODES or code in _HEAT_PUMP_CODES: + if ( + code in _ELECTRIC_ROOM_HEATER_CODES + or code in _HEAT_PUMP_CODES + or code == _HHR_STORAGE_CODE + ): return _ELECTRICITY_FUEL if code in _SOLID_FUEL_ROOM_HEATER_CODES: return _HOUSE_COAL_FUEL diff --git a/domain/epc/property_overrides/main_heating_guard.py b/domain/epc/property_overrides/main_heating_guard.py new file mode 100644 index 000000000..9eb50548b --- /dev/null +++ b/domain/epc/property_overrides/main_heating_guard.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Optional + +from domain.epc.property_overrides.main_heating_system_type import ( + MainHeatingSystemType, +) + + +def main_heating_guard(description: str) -> Optional[MainHeatingSystemType]: + """Deterministically resolve a High Heat Retention Storage Heater (HHRSH) + description, which the LLM funnelled into ``"Electric storage heaters, old"`` + (SAP 401) for want of an HHRSH archetype (#1376 / ADR-0044). + + HHRSH is a distinct RdSAP system (SAP Table 4a 409) with its own intrinsic + charge control (2404) — the *best* modern storage, not the *worst* old large- + volume type. This guard claims only the structured HHRSH phrasing and returns + ``None`` for every other description (the remaining storage subtypes the LLM + already classifies correctly, and varied phrasings), so they still reach the + LLM classifier via ``GuardedColumnClassifier``. + """ + if "high heat retention" in description.lower(): + return MainHeatingSystemType.ELECTRIC_STORAGE_HIGH_HEAT_RETENTION + return None diff --git a/domain/epc/property_overrides/main_heating_system_type.py b/domain/epc/property_overrides/main_heating_system_type.py index d79fec1d1..3d74b93c0 100644 --- a/domain/epc/property_overrides/main_heating_system_type.py +++ b/domain/epc/property_overrides/main_heating_system_type.py @@ -23,6 +23,9 @@ class MainHeatingSystemType(Enum): ELECTRIC_STORAGE_SLIMLINE = "Electric storage heaters, slimline" ELECTRIC_STORAGE_CONVECTOR = "Electric storage heaters, convector" ELECTRIC_STORAGE_FAN = "Electric storage heaters, fan" + ELECTRIC_STORAGE_HIGH_HEAT_RETENTION = ( + "Electric storage heaters, high heat retention" + ) DIRECT_ELECTRIC = "Direct-acting electric" ELECTRIC_ROOM_HEATERS = "Electric room heaters" SOLID_FUEL_ROOM_HEATER_CLOSED = "Solid fuel room heater, closed" diff --git a/scripts/reclassify_hhrsh_storage.py b/scripts/reclassify_hhrsh_storage.py new file mode 100644 index 000000000..952b205b7 --- /dev/null +++ b/scripts/reclassify_hhrsh_storage.py @@ -0,0 +1,145 @@ +"""One-time re-classification of HHRSH overrides funnelled into old storage. + +#1376 / ADR-0044: with no High Heat Retention archetype, the LLM classified +"Electric Storage Systems: High heat retention storage heaters" to +``"Electric storage heaters, old"`` (SAP 401) — scoring the *best* modern storage +as the *worst* old large-volume type. An audit found 111 such rows. + +The live classifier now applies ``main_heating_guard`` deterministically (so new +intakes are correct); this fixes the rows written before it. The **same guard** +decides the correction here, so the backfill and the live path cannot drift. + +Updates the TEXT ``property_overrides.override_value`` (what the modelling reads — +the immediate fix) unconditionally. The ``landlord_main_heating_system_overrides`` +``value`` classifier cache is a ``main_heating_system`` **pgEnum**; the new HHRSH +archetype is an FE-owned value, so its cache write is **deferred** until the +Drizzle migration adds it (the Class-A/B pattern). + +DRY-RUN BY DEFAULT: prints the counts it would change and writes nothing. Pass +``--apply`` to execute inside a transaction. Idempotent — only rows whose stored +value differs from the target member are touched. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable + +from sqlalchemy import Connection, text + +from domain.epc.property_overrides.main_heating_guard import main_heating_guard +from scripts.e2e_common import build_engine, load_env + + +def hhrsh_storage_corrections( + stored: Iterable[tuple[str, str]], +) -> dict[str, str]: + """``(description, stored override_value)`` → the HHRSH archetype value, for the + descriptions the main-heating guard resolves to HHRSH whose stored value is not + already it. Descriptions the guard defers and rows already on the target are + omitted, so re-running against corrected data is a no-op.""" + corrections: dict[str, str] = {} + for description, value in stored: + member = main_heating_guard(description) + if member is not None and value != member.value: + corrections[description] = member.value + return corrections + + +_DISTINCT = text( + """ + SELECT DISTINCT lower(original_spreadsheet_description) AS description, + override_value AS value + FROM property_overrides + WHERE override_component = 'main_heating_system' + """ +) +_OVERRIDES_UPDATE = text( + """ + UPDATE property_overrides + SET override_value = :new_value + WHERE override_component = 'main_heating_system' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) +_OVERRIDES_COUNT = text( + """ + SELECT count(*) FROM property_overrides + WHERE override_component = 'main_heating_system' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) +_CACHE_UPDATE = text( + """ + UPDATE landlord_main_heating_system_overrides + SET value = :new_value, updated_at = now() + WHERE lower(description) = :description + AND value::text <> :new_value + """ +) +_ENUM_VALUES = text( + "SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = 'main_heating_system'" +) + + +def reclassify(conn: Connection, *, apply: bool) -> tuple[int, set[str]]: + """Re-map old-storage HHRSH overrides onto the high-heat-retention archetype. + Returns the number of ``property_overrides`` rows found and the set of target + values the live ``main_heating_system`` pgEnum does not yet carry (cache- + deferred until the FE migration).""" + stored = [(r.description, r.value) for r in conn.execute(_DISTINCT)] + enum_values = {r[0] for r in conn.execute(_ENUM_VALUES)} + total = 0 + deferred: set[str] = set() + for description, new_value in hhrsh_storage_corrections(stored).items(): + params = {"description": description, "new_value": new_value} + total += conn.execute(_OVERRIDES_COUNT, params).scalar() or 0 + in_enum = new_value in enum_values + if not in_enum: + deferred.add(new_value) + if apply: + conn.execute(_OVERRIDES_UPDATE, params) + if in_enum: + conn.execute(_CACHE_UPDATE, params) + return total, deferred + + +def main() -> None: + load_env() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="execute the updates (default: dry-run, writes nothing)", + ) + args = parser.parse_args() + + engine = build_engine() + with engine.begin() as conn: + conn.execute(text("SET statement_timeout = 120000")) + total, deferred = reclassify(conn, apply=args.apply) + + verb = "re-classified" if args.apply else "would re-classify" + print( + f"{verb} {total} high-heat-retention storage override row(s) off " + "'Electric storage heaters, old' onto the HHRSH archetype (SAP 409) — " + "property_overrides / TEXT, what the modelling reads." + ) + if deferred: + print( + f"\n{len(deferred)} target value(s) NOT yet in the main_heating_system " + "pgEnum — their classifier-cache rows are deferred until the FE-repo " + "enum migration adds these members (property_overrides was still " + "updated, which is what the modelling reads):" + ) + for value in sorted(deferred): + print(f" {value!r}") + if not args.apply: + print("\nDRY-RUN — nothing written. Re-run with --apply to execute.") + + +if __name__ == "__main__": + main() diff --git a/tests/domain/epc/test_main_heating_guard.py b/tests/domain/epc/test_main_heating_guard.py new file mode 100644 index 000000000..452474d61 --- /dev/null +++ b/tests/domain/epc/test_main_heating_guard.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest + +from domain.epc.property_overrides.main_heating_guard import main_heating_guard +from domain.epc.property_overrides.main_heating_system_type import ( + MainHeatingSystemType, +) + + +def test_guard_resolves_high_heat_retention_storage() -> None: + # Arrange — the structured phrasing the LLM funnelled into "old storage" (401). + description = "Electric Storage Systems: High heat retention storage heaters" + + # Act + result = main_heating_guard(description) + + # Assert — HHRSH has its own archetype (SAP 409), never old storage (ADR-0044). + assert result is MainHeatingSystemType.ELECTRIC_STORAGE_HIGH_HEAT_RETENTION + + +@pytest.mark.parametrize( + "description", + [ + # Other storage subtypes the LLM already classifies correctly — the guard + # only rescues the HHRSH gap, leaving the rest to the fallback classifier. + "Electric Storage Systems: Old (large volume) storage heaters", + "Electric Storage Systems: Modern (slimline) storage heaters", + "Electric Storage Systems: Fan storage heaters", + # Unrelated / varied phrasings are the LLM's job. + "Gas boiler", + "", + ], +) +def test_guard_defers_non_hhrsh_descriptions_to_the_llm(description: str) -> None: + # Act + result = main_heating_guard(description) + + # Assert + assert result is None diff --git a/tests/domain/epc/test_main_heating_system_overlay.py b/tests/domain/epc/test_main_heating_system_overlay.py index 6e68cce03..b100f75a6 100644 --- a/tests/domain/epc/test_main_heating_system_overlay.py +++ b/tests/domain/epc/test_main_heating_system_overlay.py @@ -98,6 +98,35 @@ def test_storage_heater_subtypes_decode_to_their_codes( assert simulation.heating.sap_main_heating_code == code +def test_high_heat_retention_storage_decodes_to_its_own_code_not_old_storage() -> None: + # Act + simulation = main_heating_overlay_for( + "Electric storage heaters, high heat retention", 0 + ) + + # Assert — HHRSH is SAP Table 4a code 409, NOT old large-volume storage (401) + # the classifier funnelled it into (ADR-0044). + assert simulation is not None + assert simulation.heating is not None + assert simulation.heating.sap_main_heating_code == 409 + + +def test_high_heat_retention_storage_drags_its_intrinsic_companions() -> None: + # HHRSH has a single intrinsic charge control (Table 4e 2404, 0 C adjustment), + # so — unlike a generic storage heater whose control is unobserved — it must + # NOT drag the conservative manual default (2401, +0.7 C penalty), which would + # under-credit it. It is electric, on an off-peak (Dual) meter (ADR-0044). + simulation = main_heating_overlay_for( + "Electric storage heaters, high heat retention", 0 + ) + + assert simulation is not None + assert simulation.heating is not None + assert simulation.heating.main_heating_control == 2404 + assert simulation.heating.main_fuel_type == 29 + assert simulation.heating.meter_type == "Dual" + + @pytest.mark.parametrize( "main_heating_value", [ diff --git a/tests/scripts/test_reclassify_hhrsh_storage.py b/tests/scripts/test_reclassify_hhrsh_storage.py new file mode 100644 index 000000000..c3b025c53 --- /dev/null +++ b/tests/scripts/test_reclassify_hhrsh_storage.py @@ -0,0 +1,70 @@ +"""The HHRSH reclassify maps the "old storage" dumping ground onto the high-heat- +retention archetype, leaves other storage subtypes alone, and is idempotent +(#1376 / ADR-0044).""" + +from __future__ import annotations + +from domain.epc.property_overlays.main_heating_system_overlay import ( + main_heating_overlay_for, +) +from domain.epc.property_overrides.main_heating_guard import main_heating_guard +from domain.epc.property_overrides.main_heating_system_type import ( + MainHeatingSystemType, +) +from scripts.reclassify_hhrsh_storage import hhrsh_storage_corrections + + +def test_hhrsh_rows_are_remapped_off_old_storage() -> None: + # Arrange — HHRSH funnelled into old storage (the bug), plus a genuinely-old + # storage row and a gas row that must be left untouched. + stored = [ + ( + "electric storage systems: high heat retention storage heaters", + "Electric storage heaters, old", + ), + ( + "electric storage systems: old (large volume) storage heaters", + "Electric storage heaters, old", + ), + ("gas boiler", "Gas boiler, regular"), + ] + + # Act + corrections = hhrsh_storage_corrections(stored) + + # Assert — only the HHRSH row moves, to its own archetype. + assert corrections == { + "electric storage systems: high heat retention storage heaters": ( + "Electric storage heaters, high heat retention" + ) + } + + +def test_every_guard_target_round_trips_to_a_resolvable_archetype() -> None: + # The HHRSH archetype the guard emits must decode to a real SAP heating code — + # a guard/overlay drift can't silently write an unmodellable value (ADR-0044). + member = main_heating_guard( + "Electric Storage Systems: High heat retention storage heaters" + ) + + assert member is not None + assert member in MainHeatingSystemType + simulation = main_heating_overlay_for(member.value, 0) + assert simulation is not None + assert simulation.heating is not None + assert simulation.heating.sap_main_heating_code is not None + + +def test_already_corrected_rows_need_no_change() -> None: + # Act / Assert — idempotent. + assert ( + hhrsh_storage_corrections( + [ + ( + "electric storage systems: high heat retention storage heaters", + "Electric storage heaters, high heat retention", + ) + ] + ) + == {} + )