"""Field-level systematic-bias scan: mean Δ(SAP) grouped by each mapped
attribute. A subgroup with large |meanΔ| AND enough n is a systematic mapping
error for that attribute (would cancel in the overall ~0 bias)."""
from __future__ import annotations
import json, statistics
from collections import defaultdict
from datatypes.epc.domain.mapper import EpcPropertyDataMapper
from domain.sap10_calculator.calculator import calculate_sap_from_inputs
from domain.sap10_calculator.rdsap.cert_to_inputs import SAP_10_2_SPEC_PRICES, cert_to_inputs
CORPUS="backend/epc_api/json_samples/RdSAP-Schema-21.0.1/corpus.jsonl"
G=defaultdict(lambda: defaultdict(list))
def gv(x): return x.get('value') if isinstance(x,dict) else x
for line in open(CORPUS):
    d=json.loads(line); L=d.get('energy_rating_current')
    if L is None: continue
    try:
        epc=EpcPropertyDataMapper.from_api_response(d)
        r=calculate_sap_from_inputs(cert_to_inputs(epc,prices=SAP_10_2_SPEC_PRICES))
    except Exception: continue
    dsap=r.sap_score_continuous-L
    bp=(d.get('sap_building_parts') or [{}])[0]
    sh=d.get('sap_heating') or {}; md=(sh.get('main_heating_details') or [{}])[0]
    G['wall_con'][bp.get('wall_construction')].append(dsap)
    G['wall_ins'][bp.get('wall_insulation_type')].append(dsap)
    G['wall_con+ins'][(bp.get('wall_construction'),bp.get('wall_insulation_type'))].append(dsap)
    G['roof_con'][bp.get('roof_construction')].append(dsap)
    G['floor_hl'][bp.get('floor_heat_loss')].append(dsap)
    G['age'][bp.get('construction_age_band')].append(dsap)
    G['main_cat'][md.get('main_heating_category')].append(dsap)
    G['whc'][sh.get('water_heating_code')].append(dsap)
    G['meter'][(d.get('sap_energy_source') or {}).get('meter_type')].append(dsap)
    G['country'][d.get('country_code')].append(dsap)
    G['dwelling'][d.get('dwelling_type')].append(dsap)
    G['nbp'][len(d.get('sap_building_parts') or [])].append(dsap)
    G['glz'][tuple(sorted(set(w.get('glazing_type') for w in (d.get('sap_windows') or []))))[:1]].append(dsap)
print("=== subgroups with |meanΔ| >= 0.5 and n >= 8 (systematic-bias candidates) ===")
hits=[]
for field, groups in G.items():
    for key, vals in groups.items():
        if len(vals) >= 8:
            m=statistics.mean(vals)
            if abs(m) >= 0.5:
                hits.append((abs(m), field, key, len(vals), m, statistics.median(vals)))
for am,field,key,n,m,med in sorted(hits, reverse=True):
    print(f"  {field:12s} = {str(key):20s} n={n:4d} meanΔ={m:+.2f} medianΔ={med:+.2f}")
if not hits: print("  (none — no systematic subgroup bias)")