diff --git a/scripts/reclassify_party_ceiling_roofs.py b/scripts/reclassify_party_ceiling_roofs.py new file mode 100644 index 000000000..3727fa7ef --- /dev/null +++ b/scripts/reclassify_party_ceiling_roofs.py @@ -0,0 +1,132 @@ +"""One-time re-classification of party-ceiling roof overrides mis-read as roofs. + +#1376: a landlord roof description that is a **party-ceiling** marker ("another / +same dwelling or premises above") was occasionally classified by the LLM to an +external ``Pitched, N mm loft insulation`` value when it carried a trailing depth — +inventing roof heat loss where a party ceiling has ~0 (RdSAP 10 Table 18: "There +is no heat loss through the roof of a building part that has the same dwelling or +another dwelling above"). ~106 ``property_overrides`` rows (party-ceiling markers +on any non-party-ceiling value), inconsistent with the ~13k of the same family +already resolving to the party-ceiling member. + +The live classifier now applies ``roof_party_ceiling_guard`` deterministically +(so new intakes are correct); this fixes the rows written before it. The **same +guard** decides the correction here, so the backfill and the live path cannot +drift. + +Updates the TEXT ``property_overrides.override_value`` (what the modelling reads — +the actual fix) and the ``landlord_roof_type_overrides.value`` classifier cache. +The party-ceiling members already exist in the roof pgEnum (13k rows store them), +so no FE migration is needed. + +DRY-RUN BY DEFAULT: prints the row count it would change and writes nothing. Pass +``--apply`` to execute inside a transaction. Idempotent — only rows whose stored +value differs from the guard's member are touched, so a second run is a no-op. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable + +from sqlalchemy import Connection, text + +from domain.epc.property_overrides.roof_party_ceiling_guard import ( + roof_party_ceiling_guard, +) +from scripts.e2e_common import build_engine, load_env + + +def party_ceiling_corrections( + stored: Iterable[tuple[str, str]], +) -> dict[str, str]: + """``(description, stored override_value)`` → the corrected value, for the + descriptions the party-ceiling guard resolves whose stored value is not already + the guard's member. Descriptions the guard leaves to the LLM (``None``) and + rows already on the right member are omitted — so the result is exactly the set + to fix, and re-running against corrected data yields an empty map (idempotent). + """ + corrections: dict[str, str] = {} + for description, value in stored: + member = roof_party_ceiling_guard(description) + if member is not None and value != member.value: + corrections[description] = member.value + return corrections + + +_DISTINCT = text( + """ + SELECT DISTINCT lower(original_spreadsheet_description) AS description, + override_value AS value + FROM property_overrides + WHERE override_component = 'roof_type' + """ +) +_OVERRIDES_UPDATE = text( + """ + UPDATE property_overrides + SET override_value = :new_value + WHERE override_component = 'roof_type' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) +_CACHE_UPDATE = text( + """ + UPDATE landlord_roof_type_overrides + SET value = :new_value, updated_at = now() + WHERE lower(description) = :description + AND value::text <> :new_value + """ +) +_OVERRIDES_COUNT = text( + """ + SELECT count(*) FROM property_overrides + WHERE override_component = 'roof_type' + AND lower(original_spreadsheet_description) = :description + AND override_value <> :new_value + """ +) + + +def reclassify(conn: Connection, *, apply: bool) -> int: + """NULL-free re-map: set every mis-classified party-ceiling roof override to the + guard's member. Returns the number of ``property_overrides`` rows found (that + ``--apply`` would / did correct).""" + stored = [(r.description, r.value) for r in conn.execute(_DISTINCT)] + total = 0 + for description, new_value in party_ceiling_corrections(stored).items(): + params = {"description": description, "new_value": new_value} + total += conn.execute(_OVERRIDES_COUNT, params).scalar() or 0 + if apply: + conn.execute(_OVERRIDES_UPDATE, params) + conn.execute(_CACHE_UPDATE, params) + return total + + +def main() -> None: + load_env() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="execute the updates (default: dry-run, writes nothing)", + ) + args = parser.parse_args() + + engine = build_engine() + with engine.begin() as conn: + conn.execute(text("SET statement_timeout = 120000")) + total = reclassify(conn, apply=args.apply) + + verb = "re-classified" if args.apply else "would re-classify" + print( + f"{verb} {total} party-ceiling roof override row(s) from an external " + "roof value to the party-ceiling member." + ) + if not args.apply: + print("\nDRY-RUN — nothing written. Re-run with --apply to execute.") + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/test_reclassify_party_ceiling_roofs.py b/tests/scripts/test_reclassify_party_ceiling_roofs.py new file mode 100644 index 000000000..513e510d5 --- /dev/null +++ b/tests/scripts/test_reclassify_party_ceiling_roofs.py @@ -0,0 +1,33 @@ +"""The party-ceiling roof reclassify corrects only the mis-read rows, to the +guard's member, and is idempotent (#1376).""" + +from __future__ import annotations + +from scripts.reclassify_party_ceiling_roofs import party_ceiling_corrections + + +def test_only_misclassified_party_ceiling_rows_are_corrected() -> None: + # Arrange — a party ceiling mis-read as an external pitched roof (the bug), + # one already on the correct member, and a genuine pitched roof. + stored = [ + ("another dwelling above: 100mm", "Pitched, 100 mm loft insulation"), + ("another dwelling above: unknown", "(another dwelling above)"), + ("pitched, normal loft access: 150mm", "Pitched, 150 mm loft insulation"), + ] + + # Act + corrections = party_ceiling_corrections(stored) + + # Assert — only the mis-read party ceiling is corrected, to its member value; + # the already-correct row and the genuine pitched roof are left alone. + assert corrections == { + "another dwelling above: 100mm": "(another dwelling above)" + } + + +def test_already_corrected_data_yields_no_further_corrections() -> None: + # Arrange — a re-run over data the first pass already fixed. + stored = [("another dwelling above: 100mm", "(another dwelling above)")] + + # Act / Assert — idempotent: nothing left to change. + assert party_ceiling_corrections(stored) == {}