From 1424e2110ecad68685fa3927a7a82a1f415c9ce3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 19 Jul 2026 11:40:51 +0000 Subject: [PATCH] Re-pin Wythenshawe accuracy harness to the DB oracle (#1649) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_pashub_sap_accuracy_wythenshawe` graded against the manifest `pre_sap` (`hubspot_deal_data.pre_sap`) — surveyor hand-entered and frequently wrong, which floated a misleading 79% / MAE 0.75. Grade instead against the DB PasHub-assessment SAP (`epc_property_energy_performance.energy_rating_current`, `ep.source='lodged'`), matched by UPRN or postcode+house-number. `scripts/enrich_pashub_wythenshawe_oracle.py` resolves each fixture's oracle SAP once (DB required, not run in CI) and bakes `oracle_sap` / `oracle_match` into `manifest.json`, so the harness stays DB-free. 148/168 resolve (39 UPRN + 109 address); the 20 unmatched carry `oracle_sap` null and are excluded from the aggregate until PasHub sources their SAP. With the six from_site_notes fixes, the 148 matched score within-0.5 89.2% (132/148), MAE 0.297 (was, vs pre_sap, 79.2% / 0.754). Ratchets re-baselined 0.75→0.88 floor, 0.85→0.31 ceiling. The residual tail is the integer-oracle rounding floor (spec-correct dwellings) plus a couple of spec-literal gaps, not engine defects. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../pashub_accuracy_wythenshawe/manifest.json | 672 +++++++++++++----- .../test_pashub_sap_accuracy_wythenshawe.py | 108 +-- scripts/enrich_pashub_wythenshawe_oracle.py | 134 ++++ 3 files changed, 700 insertions(+), 214 deletions(-) create mode 100644 scripts/enrich_pashub_wythenshawe_oracle.py diff --git a/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json b/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json index ac81df84e..c1a601e57 100644 --- a/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json +++ b/backend/documents_parser/tests/fixtures/pashub_accuracy_wythenshawe/manifest.json @@ -8,1176 +8,1512 @@ "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500671707334.pdf" + "pdf": "500671707334.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500671707335", "uprn": "000077046521", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500671707335.pdf" + "pdf": "500671707335.pdf", + "oracle_sap": 64, + "oracle_match": "uprn" }, { "deal_id": "500671707336", "uprn": "000077053410", "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500671707336.pdf" + "pdf": "500671707336.pdf", + "oracle_sap": 59, + "oracle_match": "uprn" }, { "deal_id": "500671707337", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500671707337.pdf" + "pdf": "500671707337.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500677104834", "uprn": "000077050547", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500677104834.pdf" + "pdf": "500677104834.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500717664451", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500717664451.pdf" + "pdf": "500717664451.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500717664452", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500717664452.pdf" + "pdf": "500717664452.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500717664453", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500717664453.pdf" + "pdf": "500717664453.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500717664454", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500717664454.pdf" + "pdf": "500717664454.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500717664455", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500717664455.pdf" + "pdf": "500717664455.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500719659230", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500719659230.pdf" + "pdf": "500719659230.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500719659233", "uprn": null, "pre_sap_raw": "D56", "pre_sap_score": 56, - "pdf": "500719659233.pdf" + "pdf": "500719659233.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500719659234", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500719659234.pdf" + "pdf": "500719659234.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500719659235", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500719659235.pdf" + "pdf": "500719659235.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500721469635", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500721469635.pdf" + "pdf": "500721469635.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500721469636", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500721469636.pdf" + "pdf": "500721469636.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500738218187", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500738218187.pdf" + "pdf": "500738218187.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500738218188", "uprn": "000077073680", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500738218188.pdf" + "pdf": "500738218188.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500738218190", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500738218190.pdf" + "pdf": "500738218190.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500752190667", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500752190667.pdf" + "pdf": "500752190667.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500752190668", "uprn": "000077051200", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500752190668.pdf" + "pdf": "500752190668.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500752190669", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500752190669.pdf" + "pdf": "500752190669.pdf", + "oracle_sap": 59, + "oracle_match": "address" }, { "deal_id": "500752190670", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500752190670.pdf" + "pdf": "500752190670.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500752190671", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500752190671.pdf" + "pdf": "500752190671.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500752190672", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500752190672.pdf" + "pdf": "500752190672.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500759637226", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500759637226.pdf" + "pdf": "500759637226.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500761275639", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500761275639.pdf" + "pdf": "500761275639.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500761275640", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500761275640.pdf" + "pdf": "500761275640.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500762478811", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500762478811.pdf" + "pdf": "500762478811.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500762478813", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500762478813.pdf" + "pdf": "500762478813.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500764850382", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500764850382.pdf" + "pdf": "500764850382.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500764850384", "uprn": "000077052680", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500764850384.pdf" + "pdf": "500764850384.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500764850386", "uprn": "000077051448", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500764850386.pdf" + "pdf": "500764850386.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500764850389", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500764850389.pdf" + "pdf": "500764850389.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500766759115", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500766759115.pdf" + "pdf": "500766759115.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500766759116", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500766759116.pdf" + "pdf": "500766759116.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500766759117", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500766759117.pdf" + "pdf": "500766759117.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500766759119", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500766759119.pdf" + "pdf": "500766759119.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500766759121", "uprn": "000077045339", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500766759121.pdf" + "pdf": "500766759121.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500767954150", "uprn": "000077046582", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500767954150.pdf" + "pdf": "500767954150.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500767954151", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500767954151.pdf" + "pdf": "500767954151.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500767954152", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500767954152.pdf" + "pdf": "500767954152.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500767954153", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500767954153.pdf" + "pdf": "500767954153.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500767954154", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500767954154.pdf" + "pdf": "500767954154.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500767954156", "uprn": null, "pre_sap_raw": "F30", "pre_sap_score": 30, - "pdf": "500767954156.pdf" + "pdf": "500767954156.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500770361585", "uprn": null, "pre_sap_raw": "D57", "pre_sap_score": 57, - "pdf": "500770361585.pdf" + "pdf": "500770361585.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500770361586", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500770361586.pdf" + "pdf": "500770361586.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500770361587", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500770361587.pdf" + "pdf": "500770361587.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500771973359", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500771973359.pdf" + "pdf": "500771973359.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500771973360", "uprn": "000077051178", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500771973360.pdf" + "pdf": "500771973360.pdf", + "oracle_sap": 64, + "oracle_match": "uprn" }, { "deal_id": "500771973361", "uprn": "000077045449", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500771973361.pdf" + "pdf": "500771973361.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500771973362", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500771973362.pdf" + "pdf": "500771973362.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500771973363", "uprn": "000077044951", "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500771973363.pdf" + "pdf": "500771973363.pdf", + "oracle_sap": 70, + "oracle_match": "uprn" }, { "deal_id": "500771973364", "uprn": "000077053171", "pre_sap_raw": "D57", "pre_sap_score": 57, - "pdf": "500771973364.pdf" + "pdf": "500771973364.pdf", + "oracle_sap": 57, + "oracle_match": "uprn" }, { "deal_id": "500771973365", "uprn": "000077050562", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500771973365.pdf" + "pdf": "500771973365.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500771973366", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500771973366.pdf" + "pdf": "500771973366.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500771973367", "uprn": null, "pre_sap_raw": "E46", "pre_sap_score": 46, - "pdf": "500771973367.pdf" + "pdf": "500771973367.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500774016249", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500774016249.pdf" + "pdf": "500774016249.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500796290249", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796290249.pdf" + "pdf": "500796290249.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500796290250", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500796290250.pdf" + "pdf": "500796290250.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500796377309", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796377309.pdf" + "pdf": "500796377309.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500796377310", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796377310.pdf" + "pdf": "500796377310.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500796377311", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500796377311.pdf" + "pdf": "500796377311.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500796377312", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796377312.pdf" + "pdf": "500796377312.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500796377314", "uprn": "000077042251", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796377314.pdf" + "pdf": "500796377314.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500796377316", "uprn": "000077053166", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377316.pdf" + "pdf": "500796377316.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500796377317", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500796377317.pdf" + "pdf": "500796377317.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500796377318", "uprn": null, "pre_sap_raw": "E53", "pre_sap_score": 53, - "pdf": "500796377318.pdf" + "pdf": "500796377318.pdf", + "oracle_sap": 53, + "oracle_match": "address" }, { "deal_id": "500796377319", "uprn": "000077053551", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377319.pdf" + "pdf": "500796377319.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500796377320", "uprn": "000077051205", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500796377320.pdf" + "pdf": "500796377320.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500796481781", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500796481781.pdf" + "pdf": "500796481781.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500796646641", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500796646641.pdf" + "pdf": "500796646641.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500796646642", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500796646642.pdf" + "pdf": "500796646642.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500796646644", "uprn": null, "pre_sap_raw": "F34", "pre_sap_score": 34, - "pdf": "500796646644.pdf" + "pdf": "500796646644.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500796700866", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500796700866.pdf" + "pdf": "500796700866.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500796700867", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500796700867.pdf" + "pdf": "500796700867.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500796700868", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500796700868.pdf" + "pdf": "500796700868.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797048053", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797048053.pdf" + "pdf": "500797048053.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797048054", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797048054.pdf" + "pdf": "500797048054.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200599", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797200599.pdf" + "pdf": "500797200599.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200604", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797200604.pdf" + "pdf": "500797200604.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797200605", "uprn": "000077080305", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797200605.pdf" + "pdf": "500797200605.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797312247", "uprn": "000077050541", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797312247.pdf" + "pdf": "500797312247.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797312248", "uprn": "000077053227", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797312248.pdf" + "pdf": "500797312248.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500797312249", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797312249.pdf" + "pdf": "500797312249.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797313211", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797313211.pdf" + "pdf": "500797313211.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500797383868", "uprn": "000077045425", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797383868.pdf" + "pdf": "500797383868.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500797383869", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500797383869.pdf" + "pdf": "500797383869.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500797383870", "uprn": "000077051737", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500797383870.pdf" + "pdf": "500797383870.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500797465814", "uprn": null, "pre_sap_raw": "E41", "pre_sap_score": 41, - "pdf": "500797465814.pdf" + "pdf": "500797465814.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797523170", "uprn": null, "pre_sap_raw": "C72", "pre_sap_score": 72, - "pdf": "500797523170.pdf" + "pdf": "500797523170.pdf", + "oracle_sap": 72, + "oracle_match": "address" }, { "deal_id": "500797523171", "uprn": "000077050551", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797523171.pdf" + "pdf": "500797523171.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500797523172", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797523172.pdf" + "pdf": "500797523172.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797523173", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797523173.pdf" + "pdf": "500797523173.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797523174", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500797523174.pdf" + "pdf": "500797523174.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500797668540", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500797668540.pdf" + "pdf": "500797668540.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500797738231", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797738231.pdf" + "pdf": "500797738231.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797738233", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500797738233.pdf" + "pdf": "500797738233.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500797739194", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500797739194.pdf" + "pdf": "500797739194.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797739195", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500797739195.pdf" + "pdf": "500797739195.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500797739196", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739196.pdf" + "pdf": "500797739196.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739197", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739197.pdf" + "pdf": "500797739197.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739198", "uprn": "000077051347", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739198.pdf" + "pdf": "500797739198.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500797739199", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797739199.pdf" + "pdf": "500797739199.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797739200", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797739200.pdf" + "pdf": "500797739200.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797805805", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500797805805.pdf" + "pdf": "500797805805.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500797805807", "uprn": null, "pre_sap_raw": "F31", "pre_sap_score": 31, - "pdf": "500797805807.pdf" + "pdf": "500797805807.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500797805809", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797805809.pdf" + "pdf": "500797805809.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500797805810", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797805810.pdf" + "pdf": "500797805810.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500797805811", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797805811.pdf" + "pdf": "500797805811.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797875440", "uprn": "000077050090", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500797875440.pdf" + "pdf": "500797875440.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500797962466", "uprn": null, "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797962466.pdf" + "pdf": "500797962466.pdf", + "oracle_sap": 60, + "oracle_match": "address" }, { "deal_id": "500797962469", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797962469.pdf" + "pdf": "500797962469.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797962470", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797962470.pdf" + "pdf": "500797962470.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797962471", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797962471.pdf" + "pdf": "500797962471.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500797995201", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500797995201.pdf" + "pdf": "500797995201.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500797995202", "uprn": "000077073032", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500797995202.pdf" + "pdf": "500797995202.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500797995203", "uprn": "000077051983", "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500797995203.pdf" + "pdf": "500797995203.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500798076127", "uprn": "000077046459", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500798076127.pdf" + "pdf": "500798076127.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500798076128", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500798076128.pdf" + "pdf": "500798076128.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500798076130", "uprn": null, "pre_sap_raw": "D68", "pre_sap_score": 68, - "pdf": "500798076130.pdf" + "pdf": "500798076130.pdf", + "oracle_sap": 68, + "oracle_match": "address" }, { "deal_id": "500798148835", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798148835.pdf" + "pdf": "500798148835.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500798148836", "uprn": "000077045469", "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798148836.pdf" + "pdf": "500798148836.pdf", + "oracle_sap": 67, + "oracle_match": "uprn" }, { "deal_id": "500798148837", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500798148837.pdf" + "pdf": "500798148837.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500798148839", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500798148839.pdf" + "pdf": "500798148839.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500798148841", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798148841.pdf" + "pdf": "500798148841.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798341369", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798341369.pdf" + "pdf": "500798341369.pdf", + "oracle_sap": 65, + "oracle_match": "address" }, { "deal_id": "500798342331", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500798342331.pdf" + "pdf": "500798342331.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500798527676", "uprn": "000077050055", "pre_sap_raw": "D60", "pre_sap_score": 60, - "pdf": "500798527676.pdf" + "pdf": "500798527676.pdf", + "oracle_sap": 60, + "oracle_match": "uprn" }, { "deal_id": "500798527677", "uprn": "000077051819", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500798527677.pdf" + "pdf": "500798527677.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500798527678", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798527678.pdf" + "pdf": "500798527678.pdf", + "oracle_sap": 67, + "oracle_match": "address" }, { "deal_id": "500798620869", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798620869.pdf" + "pdf": "500798620869.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798620872", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500798620872.pdf" + "pdf": "500798620872.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500798620874", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798620874.pdf" + "pdf": "500798620874.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500798620876", "uprn": null, "pre_sap_raw": "D58", "pre_sap_score": 58, - "pdf": "500798620876.pdf" + "pdf": "500798620876.pdf", + "oracle_sap": 58, + "oracle_match": "address" }, { "deal_id": "500798745810", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500798745810.pdf" + "pdf": "500798745810.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500798745811", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500798745811.pdf" + "pdf": "500798745811.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500798745812", "uprn": "000077045588", "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798745812.pdf" + "pdf": "500798745812.pdf", + "oracle_sap": 63, + "oracle_match": "uprn" }, { "deal_id": "500798782667", "uprn": "000077053134", "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500798782667.pdf" + "pdf": "500798782667.pdf", + "oracle_sap": 61, + "oracle_match": "uprn" }, { "deal_id": "500798782668", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500798782668.pdf" + "pdf": "500798782668.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801592506", "uprn": null, "pre_sap_raw": "C70", "pre_sap_score": 70, - "pdf": "500801592506.pdf" + "pdf": "500801592506.pdf", + "oracle_sap": 70, + "oracle_match": "address" }, { "deal_id": "500801592509", "uprn": null, "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500801592509.pdf" + "pdf": "500801592509.pdf", + "oracle_sap": 59, + "oracle_match": "address" }, { "deal_id": "500801665243", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801665243.pdf" + "pdf": "500801665243.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801665245", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500801665245.pdf" + "pdf": "500801665245.pdf", + "oracle_sap": 69, + "oracle_match": "address" }, { "deal_id": "500801726702", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801726702.pdf" + "pdf": "500801726702.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801726703", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801726703.pdf" + "pdf": "500801726703.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801726704", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801726704.pdf" + "pdf": "500801726704.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801726708", "uprn": null, "pre_sap_raw": "C69", "pre_sap_score": 69, - "pdf": "500801726708.pdf" + "pdf": "500801726708.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801726709", "uprn": null, "pre_sap_raw": "C70", "pre_sap_score": 70, - "pdf": "500801726709.pdf" + "pdf": "500801726709.pdf", + "oracle_sap": 70, + "oracle_match": "address" }, { "deal_id": "500801796315", "uprn": "000077046508", "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801796315.pdf" + "pdf": "500801796315.pdf", + "oracle_sap": 66, + "oracle_match": "uprn" }, { "deal_id": "500801796316", "uprn": "000077051312", "pre_sap_raw": "D59", "pre_sap_score": 59, - "pdf": "500801796316.pdf" + "pdf": "500801796316.pdf", + "oracle_sap": 59, + "oracle_match": "uprn" }, { "deal_id": "500801796318", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801796318.pdf" + "pdf": "500801796318.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801853637", "uprn": "000077045510", "pre_sap_raw": "C72", "pre_sap_score": 72, - "pdf": "500801853637.pdf" + "pdf": "500801853637.pdf", + "oracle_sap": 72, + "oracle_match": "uprn" }, { "deal_id": "500801853638", "uprn": "000077053376", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801853638.pdf" + "pdf": "500801853638.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801853639", "uprn": null, "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500801853639.pdf" + "pdf": "500801853639.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801853640", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801853640.pdf" + "pdf": "500801853640.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801853641", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801853641.pdf" + "pdf": "500801853641.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801853642", "uprn": null, "pre_sap_raw": "D64", "pre_sap_score": 64, - "pdf": "500801853642.pdf" + "pdf": "500801853642.pdf", + "oracle_sap": 64, + "oracle_match": "address" }, { "deal_id": "500801924329", "uprn": "000077042973", "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924329.pdf" + "pdf": "500801924329.pdf", + "oracle_sap": 62, + "oracle_match": "uprn" }, { "deal_id": "500801924330", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924330.pdf" + "pdf": "500801924330.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500801924332", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801924332.pdf" + "pdf": "500801924332.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801924334", "uprn": null, "pre_sap_raw": "D66", "pre_sap_score": 66, - "pdf": "500801924334.pdf" + "pdf": "500801924334.pdf", + "oracle_sap": 66, + "oracle_match": "address" }, { "deal_id": "500801924336", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801924336.pdf" + "pdf": "500801924336.pdf", + "oracle_sap": 62, + "oracle_match": "address" }, { "deal_id": "500801964238", "uprn": null, "pre_sap_raw": "D63", "pre_sap_score": 63, - "pdf": "500801964238.pdf" + "pdf": "500801964238.pdf", + "oracle_sap": 63, + "oracle_match": "address" }, { "deal_id": "500801964241", "uprn": null, "pre_sap_raw": "D61", "pre_sap_score": 61, - "pdf": "500801964241.pdf" + "pdf": "500801964241.pdf", + "oracle_sap": 61, + "oracle_match": "address" }, { "deal_id": "500801964242", "uprn": null, "pre_sap_raw": "D67", "pre_sap_score": 67, - "pdf": "500801964242.pdf" + "pdf": "500801964242.pdf", + "oracle_sap": null, + "oracle_match": null }, { "deal_id": "500801964243", "uprn": "000077042536", "pre_sap_raw": "D65", "pre_sap_score": 65, - "pdf": "500801964243.pdf" + "pdf": "500801964243.pdf", + "oracle_sap": 65, + "oracle_match": "uprn" }, { "deal_id": "500801964244", "uprn": null, "pre_sap_raw": "D62", "pre_sap_score": 62, - "pdf": "500801964244.pdf" + "pdf": "500801964244.pdf", + "oracle_sap": 62, + "oracle_match": "address" } ] } diff --git a/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py b/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py index 350cb7bc6..ba9917ded 100644 --- a/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py +++ b/backend/documents_parser/tests/test_pashub_sap_accuracy_wythenshawe.py @@ -1,17 +1,30 @@ """End-to-end accuracy test for the pashub site-notes extractor — WYTHENSHAWE. Sibling of `test_pashub_sap_accuracy.py` (the Guinness GMCA cohort); see that -module's docstring for the full chain and the meaning of the hybrid `pre_sap` -gauge. This module points at the `pashub_accuracy_wythenshawe/` fixtures built by +module's docstring for the full chain. This module points at the +`pashub_accuracy_wythenshawe/` fixtures built by python scripts/build_pashub_accuracy_fixtures.py \\ --project-code "[Wythenshawe WH:SHF RAs - WH:SF - 240426 - 494493718748]" \\ --out-dir pashub_accuracy_wythenshawe from `hubspot_deal_data` (the project_code above) + the `rd_sap_site_note` PDFs -in S3. The ratchets below are calibrated to THIS cohort — do not copy the -Guinness constants across; each portfolio discovers its own floor/ceiling by -running. +in S3. + +ORACLE: this module grades against the **DB PasHub-assessment SAP** — the +`epc_property_energy_performance.energy_rating_current` (`ep.source='lodged'`) +that PasHub's own assessment lodged for the dwelling — NOT the manifest +`pre_sap` (`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently +wrong; grading against it gave a misleading 79.2% / MAE 0.75 while the engine +actually matched the DB in every pre_sap≠DB case). Each fixture's oracle SAP is +resolved (by UPRN or postcode+house-number) and baked into `manifest.json` as +`oracle_sap` by `scripts/enrich_pashub_wythenshawe_oracle.py`, so CI needs no DB +access. 20 fixtures have no DB-oracle match yet (`oracle_sap` null) and are +excluded from the accuracy aggregate until PasHub sources their SAP. + +The ratchets below are calibrated to THIS cohort against the DB oracle — do not +copy the Guinness constants across; each portfolio discovers its own +floor/ceiling by running. """ from __future__ import annotations @@ -35,39 +48,22 @@ _KNOWN_GAPS: tuple[type[Exception], ...] = (UnmappedSapCode, UnmappedPasHubLabel _FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pashub_accuracy_wythenshawe" _MANIFEST_PATH = _FIXTURES_DIR / "manifest.json" -# --- Ratchets (never loosen at fixed coverage), mirroring the Guinness module. -# First full run (2026-07-16): **149 computed / 19 blocked / 0 errored** -# (within-0.5 76.5% = 114/149, MAE 0.665). The 19 blocks are a spread of -# separately-ticketed mapper gaps — floor type 'Same dwelling below' (×10), -# room-in-roof gable 'None' (×2), 'No thermostatic control...' on 'Room heaters' -# (×2), plus one-off secondary-heating / secondary-fuel / room-heater / HHR -# storage-control (2404-on-402) labels — each its own field-by-field follow-up, -# not tuning. The floor/ceiling below are the observed baseline with a hair of -# headroom; treat them as a regression tripwire, NOT a cohort accuracy figure, -# and re-baseline (coverage growth is not loosening) as further mapper fixes -# unblock fixtures. -# 2026-07-16 (#1622/#1624): room-heater control 2601 unblocked the two 'No -# thermostatic control...' fixtures → **151 computed / 17 blocked** (within-0.5 -# 76.2% = 115/151, MAE 0.716; 500767954156 lands within 0.5, 500797465814 is a -# −8.8 tail entry). MAE ceiling re-baselined 0.67 → 0.72 per the -# coverage-growth convention above. -# 2026-07-16 (#1622/#1625): floor type 'Same dwelling below' → "(another -# dwelling below)" party-floor suppression (gov-API code-8 precedent) unblocked -# all 10 fixtures → **161 computed / 7 blocked** (within-0.5 75.8% = 122/161, -# MAE 0.740). 7/10 newly-computed land within 0.5 — the treatment validates; -# −3.3/−5.5 tail entries dilute the aggregates. Floor 0.76 → 0.75, ceiling -# 0.72 → 0.75. -# 2026-07-16 (#1622/#1626-#1631): the remaining one-off labels (HHR-control -# 2404→2402 downgrade, 'Loose jacket' cylinder, secondary 601/609, secondary -# dual fuel, gas room-heater 603) → **165 computed / 3 blocked** (within-0.5 -# 75.2% = 124/165, MAE 0.844). The MAE step is one accepted pre_sap -# divergence: 500797805807 (90 Brookfield Gardens) scores 49.5 vs pre_sap 31 -# (+18.5) — its own 2025 accredited cert 6890-0597-0722-0196-3943 lodges the -# identical heating codes (603/26/2601) at band E, agreeing with US; the -# PasHub pre_sap instead matches the dwelling's superseded electric -# room-heater certs (2017/2023, band F). Ceiling 0.75 → 0.85. -_MIN_WITHIN_HALF: float = 0.75 -_MAX_SAP_MAE: float = 0.85 +# --- Ratchets (never loosen at fixed coverage), against the DB oracle. +# Re-pinned to the DB PasHub-assessment SAP (#1649): grading the earlier +# pre_sap-based history (which floated 75-76% / MAE 0.67-0.85 purely on +# surveyor data-entry noise) is retired — see the module docstring. +# 2026-07-19 (#1649 A-F): with the six from_site_notes fixes in place, the +# **148 DB-oracle-matched fixtures** (39 UPRN + 109 postcode+house-number; 20 +# unmatched, excluded) score **within-0.5 89.2% (132/148), MAE 0.297**. The +# residual tail is not fixable engine defects: the DB oracle is a *rounded +# integer* SAP graded against a continuous engine (a +0.5-to-band cluster of +# spec-correct filled-cavity dwellings sits at that rounding floor), plus a +# couple of spec-literal-vs-accredited-engine gaps (e.g. 6 Barry Road's +# uninsulated solid-brick alternative wall at the RdSAP Table 6 U=1.7 default). +# Treat floor/ceiling as a regression tripwire, not a target; re-baseline +# (coverage growth is not loosening) as PasHub sources the 20 unmatched SAPs. +_MIN_WITHIN_HALF: float = 0.88 +_MAX_SAP_MAE: float = 0.31 _KNOWN_GAP_REASON = ( "pashub `from_site_notes` mapper does not int-code a main-heating " @@ -82,6 +78,13 @@ class PashubFixture: pre_sap_raw: str pre_sap_score: int pdf: str + # DB PasHub-assessment SAP (the grading oracle) — + # `epc_property_energy_performance.energy_rating_current`, `source='lodged'`, + # matched by UPRN or postcode+house-number and baked into the manifest by + # `scripts/enrich_pashub_wythenshawe_oracle.py` (see the module docstring). + # None for the 20 address-unmatched fixtures (excluded from the aggregate). + oracle_sap: Optional[int] = None + oracle_match: Optional[str] = None # "uprn" | "address" | None @property def path(self) -> Path: @@ -95,7 +98,7 @@ class Outcome: blocked: bool # blocked on the known main-heating-control gap (xfail territory) reason: Optional[str] sap_continuous: Optional[float] - diff: Optional[float] # abs(our SAP - pre_sap) + diff: Optional[float] # abs(our SAP - DB oracle); None when unmatched/blocked def _load_manifest() -> list[PashubFixture]: @@ -119,7 +122,14 @@ def _evaluate(deal_id: str) -> Outcome: result = Sap10Calculator().calculate(epc) except _KNOWN_GAPS as exc: return Outcome(blocked=True, reason=str(exc), sap_continuous=None, diff=None) - diff = abs(result.sap_score_continuous - fixture.pre_sap_score) + # Grade against the DB PasHub-assessment oracle, not the surveyor-entered + # `pre_sap`. A fixture with no DB-oracle match carries diff None and is + # excluded from the accuracy aggregate (it still asserts it computes). + diff = ( + abs(result.sap_score_continuous - fixture.oracle_sap) + if fixture.oracle_sap is not None + else None + ) return Outcome( blocked=False, reason=None, @@ -141,10 +151,14 @@ def test_pashub_fixture_computes(deal_id: str) -> None: @pytest.mark.skipif(not _FIXTURES, reason="no pashub_accuracy_wythenshawe fixtures/manifest") def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> None: - """SAP within-0.5 of pashub's `pre_sap`, aggregated over computable fixtures.""" + """SAP within-0.5 of the DB PasHub-assessment oracle, aggregated over the + address/UPRN-matched fixtures. Fixtures with no DB-oracle match are counted + (`unmatched`) but excluded from the accuracy figures — they cannot be graded + until PasHub sources their SAP.""" diffs: list[float] = [] blocked = 0 errored = 0 + unmatched = 0 for deal_id in _IDS: try: outcome = _evaluate(deal_id) @@ -154,13 +168,15 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No if outcome.blocked: blocked += 1 continue - assert outcome.diff is not None + if outcome.diff is None: # computed, but no DB-oracle SAP to grade against + unmatched += 1 + continue diffs.append(outcome.diff) if not diffs: pytest.xfail( - f"no fixture computes a SAP score " - f"({blocked} blocked on the main-heating-control gap, {errored} errored); " + f"no fixture grades against the DB oracle " + f"({blocked} blocked, {unmatched} address-unmatched, {errored} errored); " f"{_KNOWN_GAP_REASON}" ) @@ -170,8 +186,8 @@ def test_pashub_sap_accuracy_aggregate(capsys: pytest.CaptureFixture[str]) -> No with capsys.disabled(): print( - f"\n[pashub WYTHENSHAWE | {n} computed / {blocked} blocked / " - f"{errored} errored]" + f"\n[pashub WYTHENSHAWE | {n} graded / {unmatched} unmatched / " + f"{blocked} blocked / {errored} errored | oracle=DB energy_rating_current]" f"\n SAP within-0.5 = {within_half:.1%} MAE = {sap_mae:.3f}" ) diff --git a/scripts/enrich_pashub_wythenshawe_oracle.py b/scripts/enrich_pashub_wythenshawe_oracle.py new file mode 100644 index 000000000..45701c055 --- /dev/null +++ b/scripts/enrich_pashub_wythenshawe_oracle.py @@ -0,0 +1,134 @@ +"""Enrich the PasHub Wythenshawe fixture manifest with the DB-oracle SAP. + +The accuracy harness must grade against the reliable oracle — the PasHub +company-assessment result stored in the DB (`epc_property_energy_performance. +energy_rating_current`, `ep.source='lodged'`) — NOT the manifest `pre_sap` +(`hubspot_deal_data.pre_sap`, surveyor hand-entered and frequently wrong). This +one-time / rebuild-able provenance script resolves each fixture to that DB SAP +and writes `oracle_sap` (int or null) + `oracle_match` ("uprn" / "address" / +null) back into `manifest.json`, so the committed manifest is what the test +consumes and CI needs no DB access. Mirrors `build_pashub_accuracy_fixtures.py`. + +Match order: manifest `uprn` (bigint, exact) → else postcode + house-number +parsed from the PDF "Property Address:" block. Latest cert per key by `ep.id`. + +Requires DB credentials (reads `backend/.env`); NOT run in CI. + + python scripts/enrich_pashub_wythenshawe_oracle.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + +from scripts.e2e_common import build_engine, load_env # noqa: E402 +from sqlalchemy import text # noqa: E402 +from sqlalchemy.engine import Connection # noqa: E402 + +from backend.documents_parser.parser import pdf_to_text_list # noqa: E402 + +_FIXTURES = ( + REPO_ROOT + / "backend" + / "documents_parser" + / "tests" + / "fixtures" + / "pashub_accuracy_wythenshawe" +) +_MANIFEST = _FIXTURES / "manifest.json" +_POSTCODE_RE = re.compile(r"^[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}$") +_HOUSENUM_RE = re.compile(r"^(\d+[A-Za-z]?)\b") + + +def _address_from_pdf(deal_id: str) -> tuple[Optional[str], Optional[str]]: + """(house_number, postcode) from the PDF "Property Address:" block.""" + tl = pdf_to_text_list((_FIXTURES / f"{deal_id}.pdf").read_bytes()) + try: + i = next(k for k, l in enumerate(tl) if l.strip() == "Property Address:") + except StopIteration: + return None, None + block = [l.strip().rstrip(",") for l in tl[i + 1 : i + 6] if l.strip()] + postcode = next((l for l in block if _POSTCODE_RE.match(l.upper())), None) + house = None + if block: + m = _HOUSENUM_RE.match(block[0]) + if m: + house = m.group(1) + return house, postcode + + +def _resolve( + conn: Connection, uprn: Optional[int], deal_id: str +) -> tuple[Optional[int], str]: + if uprn: + row = conn.execute( + text( + "select eppe.energy_rating_current from epc_property ep " + "join epc_property_energy_performance eppe " + "on eppe.epc_property_id = ep.id " + "where ep.source='lodged' and ep.uprn=:u " + "and eppe.energy_rating_current is not null " + "order by ep.id desc limit 1" + ), + {"u": int(uprn)}, + ).scalar() + if row is not None: + return int(row), "uprn" + house, postcode = _address_from_pdf(deal_id) + if house and postcode: + rows = conn.execute( + text( + "select ep.address_line_1, eppe.energy_rating_current " + "from epc_property ep join epc_property_energy_performance eppe " + "on eppe.epc_property_id = ep.id " + "where ep.source='lodged' " + "and replace(upper(ep.postcode),' ','')=:pc " + "and eppe.energy_rating_current is not null order by ep.id desc" + ), + {"pc": postcode.upper().replace(" ", "")}, + ).fetchall() + for row_pair in rows: + a1 = str(row_pair[0] or "").strip() + m = _HOUSENUM_RE.match(a1) + if m and m.group(1).lower() == house.lower(): + return int(row_pair[1]), "address" + return None, "unmatched" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + load_env(REPO_ROOT / "backend" / ".env") + engine = build_engine() + manifest = json.loads(_MANIFEST.read_text()) + + matched = 0 + with engine.connect() as conn: + for fx in manifest["fixtures"]: + oracle_sap, kind = _resolve(conn, fx.get("uprn"), fx["deal_id"]) + fx["oracle_sap"] = oracle_sap + fx["oracle_match"] = None if kind == "unmatched" else kind + if oracle_sap is not None: + matched += 1 + + n = len(manifest["fixtures"]) + print(f"resolved {matched}/{n} fixtures to a DB oracle SAP") + if args.dry_run: + print("--dry-run: manifest NOT written") + return + _MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n") + print(f"wrote {_MANIFEST}") + + +if __name__ == "__main__": + main()