diff --git a/scripts/corpus_1000/demand_cohort.py b/scripts/corpus_1000/demand_cohort.py new file mode 100644 index 000000000..cffa01e9e --- /dev/null +++ b/scripts/corpus_1000/demand_cohort.py @@ -0,0 +1,58 @@ +"""Demand-driven divergence analyzer. Flags corpus certs where the SAP miss is +driven by FABRIC/DEMAND (PE & CO2 move with SAP), not cost — the fixable class. +Tags nbp (building parts) and room-in-roof so multi-part / RR bugs surface.""" +from __future__ import annotations +import json +from pathlib import Path +from datatypes.epc.domain.mapper import EpcPropertyDataMapper +from domain.sap10_calculator.calculator import calculate_sap_from_inputs +from domain.sap10_calculator.rdsap.cert_to_inputs import ( + SAP_10_2_SPEC_PRICES, cert_to_demand_inputs, cert_to_inputs) + +CORPUS = Path("backend/epc_api/json_samples/RdSAP-Schema-21.0.1/corpus.jsonl") + +def has_rr(doc): + for bp in doc.get("sap_building_parts") or []: + r = bp.get("room_in_roof") or bp.get("rooms_in_roof") + if r: return True + for r in doc.get("roofs") or []: + if "room in roof" in (r.get("description") or "").lower(): return True + return False + +rows = [] +for line in CORPUS.read_text().splitlines(): + doc = json.loads(line) + L = doc.get("energy_rating_current"); peL = doc.get("energy_consumption_current") + co2L = doc.get("co2_emissions_current") + if L is None: continue + try: + epc = EpcPropertyDataMapper.from_api_response(doc) + r = calculate_sap_from_inputs(cert_to_inputs(epc, prices=SAP_10_2_SPEC_PRICES)) + d = calculate_sap_from_inputs(cert_to_demand_inputs(epc, prices=SAP_10_2_SPEC_PRICES)) + except Exception: + continue + dsap = r.sap_score_continuous - L + if abs(dsap) < 0.5: continue + peE = getattr(d, "primary_energy_kwh_per_m2", None) + dpe = (peE - peL) if (peE is not None and peL is not None) else None + co2E = d.co2_kg_per_yr/1000 if d.co2_kg_per_yr is not None else None + dco2 = (co2E - co2L) if (co2E is not None and co2L is not None) else None + nbp = len(doc.get("sap_building_parts") or []) + rr = has_rr(doc) + # demand-driven: PE moves same direction as SAP-miss inverse (PE up -> SAP down) + # i.e. dpe and dsap have OPPOSITE signs, and |dpe| is material + demand = (dpe is not None and abs(dpe) >= 4 and (dpe > 0) == (dsap < 0)) + rows.append((doc.get("uprn"), L, round(r.sap_score_continuous,1), round(dsap,2), + round(dpe,1) if dpe is not None else None, + round(dco2,3) if dco2 is not None else None, nbp, rr, demand)) + +demand_rows = [r for r in rows if r[8]] +demand_rows.sort(key=lambda x:-abs(x[3])) +print(f"diverging certs: {len(rows)} | demand-driven: {len(demand_rows)}") +mp = [r for r in demand_rows if r[6] > 1] +rr = [r for r in demand_rows if r[7]] +print(f" demand+multipart(nbp>1): {len(mp)} | demand+RR: {len(rr)}") +print(f"\n=== top demand-driven divergences ===") +print(f"{'uprn':13s} {'L':>3} {'E':>5} {'dSAP':>6} {'dPE':>6} {'dCO2':>6} {'nbp':>3} {'RR':>3}") +for r in demand_rows[:32]: + print(f"{str(r[0]):13s} {r[1]:3d} {r[2]:5.1f} {r[3]:+6.2f} {str(r[4]):>6} {str(r[5]):>6} {r[6]:3d} {'Y' if r[7] else '':>3}") diff --git a/scripts/corpus_1000/worklist.md b/scripts/corpus_1000/worklist.md index a7dcf8f62..758365356 100644 --- a/scripts/corpus_1000/worklist.md +++ b/scripts/corpus_1000/worklist.md @@ -16,7 +16,7 @@ Statuses: `[ ]` todo · `[x]` resolved ≤0.5 · 🔧 fix landed · ⚠ xfail en - ⚠ `10012334488` · L 72 / E 85.20 / Δ +13.20 <- SPEC-FAITHFUL: SAP code 195 = electric water-storage boiler (Table 4a p.170); §12 Rule 2 bills space+water at 7-hr off-peak (5.5p). Pure cost gap (PE/CO2 match). Lodged over-billed a storage system at peak rate. Pinned engine 85 in test_real_cert_sap_accuracy. ### C004 · Σ|Δ| 13.1 · 1 certs · `hpcdb10326/gas | solid brick | pitched 75 | ext1,sec:room heaters` -- [ ] `12109191` · L 58 / E 71.10 / Δ +13.10 +- ⛔ `12109191` · L 58 / E 71.10 / Δ +13.10 <- LODGED DATA ERROR: BP0 floor-1 area 365.67 m² (10× its 36.77 ground floor; TFA 407 for a 4-habitable mid-terrace) — a decimal misplacement. Engine faithfully computes the near-wall-less huge floor → low HTC/m² → over-rate. Garbage-in, not an engine bug. ### C005 · Σ|Δ| 10.3 · 4 certs · `h691/elec | cavity wall | (another dwelling | whw909` - ⚠ `line279` · L 81 / E 78.44 / Δ -2.56 <- same as 10093237426