From 178af2f044cc979dc71d6118acab2d5bf9f4f4b3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 14 Jul 2026 21:24:37 +0000 Subject: [PATCH] Add local modelling runner + portfolio-838 problem-properties handover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/run_pashub_modelling.py invokes the modelling_e2e handler in-process over a portfolio using STORED EPCs (refetch_epc=False — the correct source for PasHub cohorts, issue #1589), parametrised by env (portfolio/scenario/pids/ dry-run/batch). Handover documents the 3 re-extraction stragglers, the rebaseliner pass-through gotcha (effective_sap_score IS the pashub rating for unchanged lodged EPCs — never validate the calculator against it), and the 7 root-caused extractor bugs now tracked in #1590. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md | 86 ++++++++++++ scripts/run_pashub_modelling.py | 122 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md create mode 100644 scripts/run_pashub_modelling.py diff --git a/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md new file mode 100644 index 000000000..e13f81c84 --- /dev/null +++ b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md @@ -0,0 +1,86 @@ +# PasHub portfolio 838 — problem properties & validation (2026-07-14) + +Cohort: **The Guinness Partnership GMCA** (205 PasHub site-notes properties), +portfolio **838**, scenario **1297**. Modelled locally through the real +`applications/modelling_e2e` lambda handler using the **stored** PasHub site +notes (`refetch_epc=False`) — see `scripts/run_pashub_modelling.py`. + +## Headline: our SAP calculator diverges from pashub's rating by ~MAE 2.7 — real work remains + +**Do not use `property_baseline_performance.effective_sap_score` to validate the +calculator.** For a SAP-10.2 lodged EPC whose physical state is unchanged, the +rebaseliner passes **Lodged Performance through as Effective** +(`domain/property_baseline/rebaseliner.py`), and Lodged Performance's `sap_score` +is `energy_rating_current` — the fetched pashub rating +(`domain/property_baseline/performance.py:63`). So `effective_sap_score` **is** +the pashub rating; comparing it to `energy_rating_current` compares the rating to +itself (circular — an earlier "99.5% match" was this mistake). + +The real accuracy of our **calculator** is the `test_pashub_sap_accuracy` +harness: **~12.2% within-0.5, MAE ~2.7** vs `pre_sap` (which ≈ the pashub +rating, MAE 0.15 — same ground truth). Worked example: property **754881** — our +extraction is **byte-identical** to the stored ingestion (fuel 26, PCDB index +18119, wall 4/2, roof 200mm, party-wall 4, floor solid), yet our calculator +scores **71.9** where pashub lodged **85** — a genuine ~13-point gap to +root-cause. These are the extraction/calculator bugs to hunt. + +**Image stripping does NOT lose data:** the fixture (stripped) and DB (original +PDF) parses of 754881 are identical field-for-field — so the harness fixtures are +faithful and the ~2.7 MAE is a true calculator gap, not a fixture artifact. +`parse_site_notes_pdf` never extracts the SAP rating from the PDF at all +(`energy_rating_current` is populated by Dan's separate pashub-API fetch, +`pashub_service.py:45` ← `preSapRating`), which is why the rating is `None` on a +raw parse but present in the DB — unrelated to stripping. + +## Properties that did not run (need PasHub re-extraction — data, not code) + +These 3 have **no fresh int-coded stored site note** (only pre-fix rows with a +string fuel), so modelling strict-raised and skipped them. Re-trigger PasHub +extraction so a current-mapper site note is stored, then they model like the +other 201. + +| property_id | UPRN | address | stored site-note fuel | fix | +|---|---|---|---|---| +| 754772 | 77168847 | 9 Philips Park Court, Willdale Close, M11 4DH | `"Mains gas"` (string) | re-extract | +| 754780 | 77180607 | 12 Seymour Road South, Clayton, M11 4PG | `"Mains gas"` (string) | re-extract | +| 754844 | 77155031 | 16 Bingley Close, Beswick, M11 3RF | `""` (blank) | re-extract; if the survey genuinely lodges no main fuel, that is a separate blank/residual-fuel mapper gap | + +## Properties that ran but not from their own PasHub survey + +| property_id | UPRN | address | note | +|---|---|---|---| +| 754816 | 77180053 | 130 Stanton Street, Clayton, M11 4PX | **no PasHub site note exists at all** — modelled via the prediction path. Needs extraction so it models from its own survey. | +| 754778 | 77181049 | (M11) | winner lodged EPC is **not** a PasHub site note (gov EPC / other); modelled SAP 60 vs stored rating 57 (**d=+3**, the cohort's only >0.5 divergence). Check why its PasHub site note isn't the winner. | + +## Accuracy outliers surfaced by closing the extractor gaps + +Closing the last 4 `test_pashub_sap_accuracy` xfails (see the PR) makes them +**compute**; two then show a large gap **vs `pre_sap`** (not vs the stored +rating — verify `pre_sap` first, do not tune to it): + +| fixture (deal) | ours | pre_sap | note | +|---|---|---|---| +| 499584755922 | 34.0 | 67 | main fuel extracted as **Bulk LPG (27)** + house-coal secondary → low SAP. Either a bad `pre_sap` or a main-fuel extraction issue; confirm the survey's main fuel. | +| 507644414148 | 70.2 | 52 | **community heating**. The PasHub path maps only the Table 4e Group 3 control code (2306); it does **not** yet set the full heat-network fuel/flags (main_fuel is 26, not a Table 12 community code). Deeper community-heating mapping gap. | + +## Root-caused extractor bugs (issue #1590) + +Five worst-divergence properties were deep-dived (extracted inputs diffed +against the PDF text; gaps attributed via patch-and-rerun). **7 distinct +extractor bugs** — PV arrays never extracted (−13.5), roof "Insulation At: +None" treated as unknown not zero (−7.6/−7.1, systematic), `pv_connection` +string passthrough (−6.9), ventilation kind never mapped (−4.7), room-in-roof +never built (−2.1), cylinder "No Access" passthrough (−0.6), system-build/ +basement code-6 collision (latent) — are itemised with fixes and fixture deals +in **https://github.com/Hestia-Homes/Model/issues/1590**. Also there: the +ground-truth caveat that `pre_sap` is pashub's *preliminary* figure — the +accredited lodgement can differ downstream of the site notes (754917: 53 → 43). + +## How to reproduce + +``` +python scripts/run_pashub_modelling.py # dry-run, all 838 batches +RUN_DRY=0 python scripts/run_pashub_modelling.py # real writes +RUN_PIDS="754772" RUN_DRY=0 python scripts/run_pashub_modelling.py # one property +``` +Validation sweep (ours vs stored rating) is in the PR description / this handover. diff --git a/scripts/run_pashub_modelling.py b/scripts/run_pashub_modelling.py new file mode 100644 index 000000000..32dbe9f5b --- /dev/null +++ b/scripts/run_pashub_modelling.py @@ -0,0 +1,122 @@ +"""Run the modelling_e2e handler locally over a portfolio's properties, using +the STORED EPCs (``refetch_epc=False``) — the correct source for PasHub +site-notes cohorts, where the stored site note is the newest assessment (the +gov-API refetch would bypass it; see issue #1589 / ADR-0001). + +Invokes the real Lambda handler in-process (no Docker) with one SQS-shaped event +per batch of property ids. Reads ``backend/.env`` and maps ``DB_*`` -> the +``POSTGRES_*`` vars the handler's ``PostgresConfig.from_env`` expects. + +Config via environment variables (all optional): + + RUN_PORTFOLIO portfolio id (default 838) + RUN_SCENARIO scenario id (default 1297) + RUN_PIDS comma-separated property ids; unset -> every property in the + portfolio, ordered by id + RUN_DRY "1" dry-run / "0" write to the DB (default "1") + RUN_BATCH properties per handler invocation (default 50) + RUN_REFETCH_EPC / RUN_REPREDICT_EPC / RUN_REFETCH_SOLAR + "1"/"0" to override the refetch flags (default "0" — use + stored EPCs/solar; flip to re-fetch from the gov API/Google) + +Examples: + python scripts/run_pashub_modelling.py # dry-run, whole portfolio + RUN_DRY=0 python scripts/run_pashub_modelling.py # real writes + RUN_SCENARIO=1301 RUN_DRY=0 python scripts/run_pashub_modelling.py + RUN_PIDS=754772,754780 RUN_DRY=0 python scripts/run_pashub_modelling.py +""" + +from __future__ import annotations + +import json +import os + +from sqlalchemy import text + +from scripts.e2e_common import build_engine, load_env + + +def _configure_env() -> None: + """Load ``backend/.env`` and mirror the FastAPI-layer ``DB_*`` creds onto the + ``POSTGRES_*`` names the modelling handler reads.""" + load_env() + for pg, db in ( + ("POSTGRES_HOST", "DB_HOST"), + ("POSTGRES_PORT", "DB_PORT"), + ("POSTGRES_USERNAME", "DB_USERNAME"), + ("POSTGRES_PASSWORD", "DB_PASSWORD"), + ("POSTGRES_DATABASE", "DB_NAME"), + ): + if db in os.environ: + os.environ.setdefault(pg, os.environ[db]) + + +def _flag(name: str, default: bool) -> bool: + raw = os.environ.get(name) + return default if raw is None else raw == "1" + + +def _portfolio_property_ids(portfolio_id: int) -> list[int]: + engine = build_engine() + with engine.begin() as conn: + conn.execute(text("SET statement_timeout = 120000")) + rows = conn.execute( + text("SELECT id FROM property WHERE portfolio_id = :p ORDER BY id"), + {"p": portfolio_id}, + ) + return [int(r[0]) for r in rows] + + +def main() -> None: + _configure_env() + # Imported after env is configured so the handler's module-level engine binds + # to the mirrored POSTGRES_* vars. + from applications.modelling_e2e.handler import handler + + portfolio_id = int(os.environ.get("RUN_PORTFOLIO", "838")) + scenario_id = int(os.environ.get("RUN_SCENARIO", "1297")) + dry_run = _flag("RUN_DRY", True) + batch_size = int(os.environ.get("RUN_BATCH", "50")) + refetch_epc = _flag("RUN_REFETCH_EPC", False) + repredict_epc = _flag("RUN_REPREDICT_EPC", False) + refetch_solar = _flag("RUN_REFETCH_SOLAR", False) + + pids_env = os.environ.get("RUN_PIDS") + property_ids: list[int] = ( + [int(x) for x in pids_env.split(",") if x.strip()] + if pids_env + else _portfolio_property_ids(portfolio_id) + ) + batches: list[list[int]] = [ + property_ids[i : i + batch_size] + for i in range(0, len(property_ids), batch_size) + ] + + print( + f"portfolio={portfolio_id} scenario={scenario_id} dry_run={dry_run} " + f"refetch_epc={refetch_epc} repredict_epc={repredict_epc} " + f"refetch_solar={refetch_solar} properties={len(property_ids)} " + f"batches={len(batches)}" + ) + + for i, chunk in enumerate(batches): + body = { + "property_ids": chunk, + "portfolio_id": portfolio_id, + "scenario_id": scenario_id, + "refetch_epc": refetch_epc, + "repredict_epc": repredict_epc, + "refetch_solar": refetch_solar, + "dry_run": dry_run, + } + event = {"Records": [{"body": json.dumps(body)}]} + print( + f"\n=== batch {i + 1}/{len(batches)}: {len(chunk)} props " + f"({chunk[0]}..{chunk[-1]}) ===" + ) + result: object = handler(event, None) + print(" ->", json.dumps(result, default=str)[:800]) + + +if __name__ == "__main__": + main()