From 585c8b1501c1e707a7f965dfc99fd2923d8479de Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 30 Jun 2026 20:30:59 +0000 Subject: [PATCH] One-time script to re-classify stale heating overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministic catalogue remap (no LLM) that re-resolves the stored archetype for the canonical RdSAP heating descriptions the under-populated taxonomy mis-classified. Dry-run by default; --apply writes inside a transaction; idempotent. Updates property_overrides.override_value (TEXT — what the modelling reads, the actual band-G fix: 3,028 rows incl. 2,210 community boilers + 770 panel/convector room heaters). Cache (the Drizzle-owned main_heating_system pgEnum) updates are deferred per-value until the FE-repo enum migration adds the new archetypes — surfaced explicitly by the script. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/reclassify_heating_overrides.py | 176 ++++++++++++++++++ .../test_reclassify_heating_overrides.py | 30 +++ 2 files changed, 206 insertions(+) create mode 100644 scripts/reclassify_heating_overrides.py create mode 100644 tests/scripts/test_reclassify_heating_overrides.py diff --git a/scripts/reclassify_heating_overrides.py b/scripts/reclassify_heating_overrides.py new file mode 100644 index 000000000..002ebef9b --- /dev/null +++ b/scripts/reclassify_heating_overrides.py @@ -0,0 +1,176 @@ +"""One-time re-classification of stale landlord main-heating-system overrides. + +PRD #1361 Class A: the LLM landlord-description classifier ran against a +too-small `MainHeatingSystemType` taxonomy (ADR-0041), so canonical RdSAP +heating descriptions were stored against the wrong archetype — community heating +and oil/solid room heaters dumped into "Gas CPSU", and direct-acting +"panel, convector or radiant" room heaters mis-read as off-peak storage (the +band-G crater). The taxonomy is now complete, so these *canonical* descriptions +re-resolve deterministically — no LLM, no nondeterminism. + +This rewrites the resolved archetype on every affected row in BOTH the +per-Property fact layer (`property_overrides.override_value`) and the +per-portfolio classifier cache (`landlord_main_heating_system_overrides.value`), +keyed on the raw description (`original_spreadsheet_description` / `description`). + +DRY-RUN BY DEFAULT: prints the row counts it would change and writes nothing. +Pass `--apply` to execute inside a transaction. Idempotent — only rows whose +stored value differs from the corrected archetype are touched, so a second run +is a no-op. Descriptions with no confident archetype yet (solid-fuel open-fire / +with-boiler, warm air, electric underfloor) are deliberately NOT remapped here. +""" + +from __future__ import annotations + +import argparse + +from sqlalchemy import text + +from domain.epc.property_overrides.main_heating_system_type import ( + MainHeatingSystemType, +) +from scripts.e2e_common import build_engine, load_env + +# Canonical RdSAP main-heating description (lowercased, as stored in +# `original_spreadsheet_description` / `description`) → the corrected archetype. +# Only descriptions whose correct archetype now exists in the taxonomy appear +# here; the value is a `MainHeatingSystemType` so a typo can't slip a non-archetype +# string into the DB (the enum is the single source of truth — ADR-0041). +REMAP: dict[str, MainHeatingSystemType] = { + "community heating systems: community boilers only (rdsap)": ( + MainHeatingSystemType.COMMUNITY_BOILERS + ), + "community heating systems: community chp and boilers (rdsap)": ( + MainHeatingSystemType.COMMUNITY_CHP_AND_BOILERS + ), + "electric (direct acting) room heaters: panel, convector or radiant heaters": ( + MainHeatingSystemType.ELECTRIC_ROOM_HEATERS + ), + "oil room heaters: room heater, 2000 or later": ( + MainHeatingSystemType.OIL_ROOM_HEATER_POST_2000 + ), + "solid fuel room heaters: closed room heater": ( + MainHeatingSystemType.SOLID_FUEL_ROOM_HEATER_CLOSED + ), + "gas (including lpg) room heaters: condensing gas fire": ( + MainHeatingSystemType.GAS_FIRE_CONDENSING + ), + "gas (including lpg) room heaters: decorative fuel effect gas fire, " + "open to chimney": MainHeatingSystemType.GAS_FIRE_DECORATIVE, + "gas (including lpg) room heaters: flush fitting live fuel effect gas fire " + "(open fronted), sealed to fireplace opening": ( + MainHeatingSystemType.GAS_FIRE_FLUSH_LIVE_EFFECT + ), + "gas (including lpg) room heaters: gas fire, open flue, 1980 or later " + "(open fronted), sitting proud of, and sealed to, fireplace opening": ( + MainHeatingSystemType.GAS_FIRE_OPEN_FLUE_POST_1980 + ), + "gas (including lpg) room heaters: gas fire, open flue, pre-1980 " + "(open fronted)": MainHeatingSystemType.GAS_FIRE_OPEN_FLUE_PRE_1980, +} + +# property_overrides keys the raw text on `original_spreadsheet_description`; the +# classifier cache keys it on `description`. Both compared case-insensitively. +_PROPERTY_OVERRIDES_UPDATE = text( + """ + UPDATE property_overrides + SET override_value = :new_value + WHERE override_component = 'main_heating_system' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) +# The cache `value` column is the Drizzle-owned `main_heating_system` pgEnum; +# compare/update it as text (no CAST) so a target value the DB enum does not yet +# carry doesn't error the read. The UPDATE is only issued for values the live +# enum already has — `_db_enum_values` gates it (see main()). +_CACHE_UPDATE = text( + """ + UPDATE landlord_main_heating_system_overrides + SET value = :new_value, updated_at = now() + WHERE lower(description) = :description + AND value::text <> :new_value + """ +) +_PROPERTY_OVERRIDES_COUNT = text( + """ + SELECT count(*) FROM property_overrides + WHERE override_component = 'main_heating_system' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) +_CACHE_COUNT = text( + """ + SELECT count(*) FROM landlord_main_heating_system_overrides + WHERE lower(description) = :description + AND value::text <> :new_value + """ +) +_ENUM_VALUES = text( + "SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = 'main_heating_system'" +) + + +def main() -> None: + load_env() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="execute the updates (default: dry-run, writes nothing)", + ) + args = parser.parse_args() + + engine = build_engine() + with engine.begin() as conn: + conn.execute(text("SET statement_timeout = 120000")) + # The cache `value` is the Drizzle-owned pgEnum; only update it for + # archetypes the live enum already carries. The rest need the Drizzle + # migration (FE repo) before the cache — and the classifier — can store + # them; property_overrides (TEXT, what the modelling reads) is updated + # regardless, which is the actual band-G fix. + enum_values = {r[0] for r in conn.execute(_ENUM_VALUES)} + prop_total = 0 + cache_total = 0 + deferred: list[str] = [] + for description, archetype in REMAP.items(): + params = {"description": description, "new_value": archetype.value} + prop_n = conn.execute(_PROPERTY_OVERRIDES_COUNT, params).scalar() or 0 + prop_total += prop_n + in_enum = archetype.value in enum_values + cache_n = conn.execute(_CACHE_COUNT, params).scalar() or 0 if in_enum else 0 + cache_total += cache_n + if not in_enum and prop_n: + deferred.append(archetype.value) + if prop_n or cache_n: + tag = "" if in_enum else " [cache deferred: enum value missing]" + print( + f" {prop_n:>6} property_overrides + {cache_n} cache " + f"-> {archetype.value!r} <= {description!r}{tag}" + ) + if args.apply: + conn.execute(_PROPERTY_OVERRIDES_UPDATE, params) + if in_enum: + conn.execute(_CACHE_UPDATE, params) + verb = "updated" if args.apply else "would update" + print( + f"\n{verb} {prop_total} property_overrides rows + {cache_total} " + f"classifier-cache rows across {len(REMAP)} descriptions." + ) + if deferred: + print( + f"\n{len(set(deferred))} archetype(s) NOT in the Drizzle " + "`main_heating_system` pgEnum — their cache rows are deferred " + "until the FE-repo enum migration lands (property_overrides was " + "still updated, which is what the modelling reads):" + ) + for v in sorted(set(deferred)): + print(f" {v!r}") + if not args.apply: + print("\nDRY-RUN — nothing written. Re-run with --apply to execute.") + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/test_reclassify_heating_overrides.py b/tests/scripts/test_reclassify_heating_overrides.py new file mode 100644 index 000000000..f3cb73382 --- /dev/null +++ b/tests/scripts/test_reclassify_heating_overrides.py @@ -0,0 +1,30 @@ +"""The one-time heating-override re-classification map is internally valid.""" + +from __future__ import annotations + +from domain.epc.property_overlays.main_heating_system_overlay import ( + main_heating_overlay_for, +) +from domain.epc.property_overrides.main_heating_system_type import ( + MainHeatingSystemType, +) +from scripts.reclassify_heating_overrides import REMAP + + +def test_every_remap_target_is_a_resolvable_archetype() -> None: + # The remap must never point a stored description at an archetype the overlay + # can't model — that would replace one broken value with another. Every + # target must decode to a real SAP heating code (ADR-0041). + for description, archetype in REMAP.items(): + simulation = main_heating_overlay_for(archetype.value, 0) + + assert simulation is not None, description + assert simulation.heating is not None, description + assert simulation.heating.sap_main_heating_code is not None, description + + +def test_remap_targets_are_main_heating_system_members() -> None: + # Belt and braces: the values are enum members (a typo can't smuggle a + # non-archetype string into the DB). + for archetype in REMAP.values(): + assert archetype in MainHeatingSystemType