From 58125503f0acd5663a194b738f5b7f28c33c6461 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 3 Jul 2026 05:19:20 +0000
Subject: [PATCH] =?UTF-8?q?chore(corpus):=20why=5Fdiverge.py=20=E2=80=94?=
 =?UTF-8?q?=20categorize=20all=20223=20diverging=20certs=20by=20reason?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/corpus_1000/why_diverge.py | 76 ++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 scripts/corpus_1000/why_diverge.py

diff --git a/scripts/corpus_1000/why_diverge.py b/scripts/corpus_1000/why_diverge.py
new file mode 100644
index 000000000..cf2aef86b
--- /dev/null
+++ b/scripts/corpus_1000/why_diverge.py
@@ -0,0 +1,76 @@
+"""Categorize every corpus cert outside +-0.5 SAP by the REASON it diverges,
+combining the computed cost-vs-demand signature, the electric heating code /
+meter, and the ledger's RR/multipart tokens."""
+from __future__ import annotations
+import json, re
+from pathlib import Path
+from collections import defaultdict
+from datatypes.epc.domain.mapper import EpcPropertyDataMapper
+from domain.sap10_calculator.calculator import calculate_sap_from_inputs
+from domain.sap10_calculator.rdsap.cert_to_inputs import (
+    SAP_10_2_SPEC_PRICES, cert_to_demand_inputs, cert_to_inputs, _is_electric_main,
+    _first_main_heating)
+
+CORPUS = Path("backend/epc_api/json_samples/RdSAP-Schema-21.0.1/corpus.jsonl")
+# ledger signatures (for RR / multipart tokens)
+sig_of={}; cur=""
+for l in Path("scripts/corpus_1000/worklist.md").read_text().splitlines():
+    if l.startswith('### C'):
+        m=re.search(r'`([^`]*)`',l); cur=m.group(1) if m else ""
+    m=re.match(r'^- .*?`([^`]+)`',l)
+    if m: sig_of[m.group(1)]=cur
+
+def bucket(doc, dsap, dpe, dco2, code, mt, sig):
+    rr = 'RR' in (sig or '')
+    elec = code in range(191,600) if code else False
+    demand = dpe is not None and abs(dpe) >= 4 and (dpe>0)==(dsap<0)
+    cost = dpe is not None and abs(dpe) < 4
+    # 1. electric storage-boiler / CPSU on off-peak (spec §12 Rule 2/1)
+    if code in (192,193,194,195,196) and dsap>0:
+        return "elec storage/CPSU boiler → §12 off-peak (spec-faithful; lodged over-billed)"
+    # 2. electric direct + storage heaters, tariff/cost
+    if elec and cost:
+        return "electric tariff/cost (meter-column & high-rate fraction; spec-faithful vs lodged)"
+    if elec and demand:
+        return "electric + fabric/demand mismatch"
+    if elec:
+        return "electric other"
+    # 3. RR
+    if rr:
+        return "room-in-roof surface geometry (spec-mandated, worksheet-validated; high variance)"
+    # 4. non-elec demand/fabric
+    if demand:
+        return "non-electric fabric/demand (per-cert: glazing area, lodged geometry, mixed walls)"
+    if cost:
+        return "non-electric cost (secondary fuel / cylinder / tariff)"
+    return "unclassified"
+
+cnt=defaultdict(list)
+band=defaultdict(int)
+for line in CORPUS.read_text().splitlines():
+    doc=json.loads(line); L=doc.get('energy_rating_current')
+    if L is None: continue
+    try:
+        epc=EpcPropertyDataMapper.from_api_response(doc)
+        r=calculate_sap_from_inputs(cert_to_inputs(epc, prices=SAP_10_2_SPEC_PRICES))
+        d=calculate_sap_from_inputs(cert_to_demand_inputs(epc, prices=SAP_10_2_SPEC_PRICES))
+    except Exception: continue
+    dsap=r.sap_score_continuous-L
+    if abs(dsap)<0.5: continue
+    peL=doc.get('energy_consumption_current'); co2L=doc.get('co2_emissions_current')
+    dpe=(d.primary_energy_kwh_per_m2-peL) if peL is not None else None
+    dco2=(d.co2_kg_per_yr/1000-co2L) if co2L is not None else None
+    sh=doc.get('sap_heating') or {}; det=(sh.get('main_heating_details') or [{}])[0]
+    code=det.get('sap_main_heating_code'); mt=(doc.get('sap_energy_source') or {}).get('meter_type')
+    b=bucket(doc,dsap,dpe,dco2,code,mt,sig_of.get(str(doc.get('uprn')),''))
+    cnt[b].append((doc.get('uprn'),L,round(r.sap_score_continuous,1),round(dsap,1)))
+    ad=abs(dsap); band['0.5-1' if ad<1 else '1-2' if ad<2 else '2-4' if ad<4 else '4+']+=1
+
+total=sum(len(v) for v in cnt.values())
+print(f"TOTAL diverging (|Δ|≥0.5): {total}\n")
+print("by |Δ| band:", dict(band), "\n")
+print("=== by REASON (count, mean|Δ|, examples) ===")
+for b,rows in sorted(cnt.items(), key=lambda x:-len(x[1])):
+    mabs=sum(abs(x[3]) for x in rows)/len(rows)
+    ex=", ".join(f"{x[0]}({x[3]:+.1f})" for x in sorted(rows,key=lambda x:-abs(x[3]))[:3])
+    print(f"\n[{len(rows):3d}] mean|Δ|={mabs:.1f}  {b}\n       e.g. {ex}")