Model/scripts/audit/neighbour_divergence.py

"""Flag neighbouring dwellings that were modelled *differently despite looking
the same* — the SAP half of the "neighbours should agree" heuristic.

Motivation (portfolio 814): Khalim found a dwelling that gets an HHRSH bundle
while its next-door neighbour of the same type does not. Neighbours in one
postcode, of the same property type and built form, are usually near-identical
stock; a big split in their *effective baseline SAP* is a strong tell that one of
the pair was mis-mapped, mis-overridden, or mis-rebaselined — and a SAP split is
what then drives a recommendation split. This script surfaces those pairs
cheaply so the deep-dive (``run_modelling_e2e`` on both neighbours) can compare
their actual measure sets and root-cause the divergence.

It reaches only ``property`` + ``epc_property`` + ``property_baseline_performance``,
all via the indexed ``property_id`` — it NEVER touches the 26m-row
``recommendation`` table, so it is safe to run portfolio-wide (unlike the
measure-set comparison, which the skill does per flagged cohort in the deep-dive).

Run:
    python -m scripts.audit.neighbour_divergence --portfolio 814
    python -m scripts.audit.neighbour_divergence --portfolio 814 --min-gap 15

Writes ``neighbour_divergence.md`` + ``neighbour_divergence.csv`` and prints a
summary. Read-only: never writes to the DB.

A cohort is the set of a portfolio's properties sharing (postcode, property_type,
built_form, main-heating fuel class). Within a cohort of >= 2, any member whose
effective SAP is >= ``--min-gap`` points from the cohort median is flagged.
Heating **fuel** is part of the key because electricity scores materially lower
in SAP than mains gas for identical fabric — mixing fuels in one cohort produces
cross-fuel false outliers (an electric dwelling flagged only for being electric
among gas neighbours). Keying on fuel ensures like-vs-like, so every flagged
divergence is a same-fuel one worth investigating (on portfolio 814 this refined
20 raw divergences to 17 same-fuel comparisons). Floor area is reported, not
keyed on, so the reviewer can discount a genuinely bigger/smaller neighbour;
promoting an area guard into the cohort key is a clean future change once the
false-positive rate is measured on a real portfolio (skill Phase 6).
"""

from __future__ import annotations

import argparse
import csv
from dataclasses import dataclass
from statistics import median
from typing import Optional

from sqlalchemy import text

from scripts.e2e_common import build_engine, load_env

# Hard ceiling so a bad plan aborts instead of saturating the shared DB, mirroring
# scripts/audit/anomalies.py. Every table here is reached via the indexed
# property_id and scoped to one portfolio, so this is a backstop, not a crutch.
_STATEMENT_TIMEOUT_MS = 120_000

# Default SAP gap (points from the cohort median) at which a neighbour is flagged.
# 12 is a starting threshold, not a tuned one — a full band is ~10-11 SAP points,
# so >= 12 means the neighbour sits a clear band apart from otherwise-identical
# stock. Re-justify against a real portfolio's distribution before trusting it
# (skill Phase 6); expose it as --min-gap so a run can sweep the threshold.
_DEFAULT_MIN_GAP = 12.0


@dataclass(frozen=True)
class Neighbour:
    """One modelled property placed in its postcode/type/form cohort."""

    property_id: int
    uprn: Optional[int]
    postcode: Optional[str]
    property_type: Optional[str]
    built_form: Optional[str]
    floor_area_m2: Optional[float]
    effective_sap: Optional[float]
    effective_band: Optional[str]
    main_fuel_type: Optional[int]  # gov-API main-heating fuel code (None if unmapped)


@dataclass(frozen=True)
class Divergence:
    property_id: int
    uprn: Optional[int]
    cohort_key: str
    cohort_size: int
    detail: str


# DISTINCT ON picks the latest ingested epc_property row per property (its
# property_id is NOT unique — ingestion keeps history), ordered by id DESC.
_QUERY = text(
    """
    SELECT p.id, p.uprn, p.postcode,
           ep.property_type, ep.built_form, ep.total_floor_area_m2,
           pbp.effective_sap_score, pbp.effective_epc_band,
           mh.main_fuel_type
    FROM property p
    JOIN property_baseline_performance pbp ON pbp.property_id = p.id
    LEFT JOIN LATERAL (
        SELECT id, property_type, built_form, total_floor_area_m2
        FROM epc_property e
        WHERE e.property_id = p.id
        ORDER BY e.id DESC
        LIMIT 1
    ) ep ON TRUE
    LEFT JOIN LATERAL (
        SELECT main_fuel_type
        FROM epc_main_heating_detail m
        WHERE m.epc_property_id = ep.id
        ORDER BY m.id
        LIMIT 1
    ) mh ON TRUE
    WHERE p.portfolio_id = :portfolio_id
    ORDER BY p.id
    """
)


def _load(portfolio_id: int) -> list[Neighbour]:
    engine = build_engine()
    out: list[Neighbour] = []
    with engine.connect() as conn:
        conn.execute(text(f"SET statement_timeout = {_STATEMENT_TIMEOUT_MS}"))
        for row in conn.execute(_QUERY, {"portfolio_id": portfolio_id}):
            m = row._mapping  # noqa: SLF001 — SQLAlchemy row mapping, mirrors anomalies.py
            out.append(
                Neighbour(
                    property_id=m["id"],
                    uprn=m["uprn"],
                    postcode=m["postcode"],
                    property_type=m["property_type"],
                    built_form=m["built_form"],
                    floor_area_m2=m["total_floor_area_m2"],
                    effective_sap=m["effective_sap_score"],
                    effective_band=m["effective_epc_band"],
                    main_fuel_type=_coerce_fuel(m["main_fuel_type"]),
                )
            )
    return out


def _coerce_fuel(raw: object) -> Optional[int]:
    """The main-heating fuel code as an int, or None. ``main_fuel_type`` is a
    JSONB column (int or str depending on the source mapper), so coerce defensively
    — a non-numeric value groups as "unmapped" rather than raising."""
    if raw is None:
        return None
    try:
        return int(raw)  # type: ignore[arg-type]
    except (TypeError, ValueError):
        return None


# Electricity main-heating fuel codes (gov-API / SAP Table 12): standard tariff
# (30), the off-peak tariffs (31-40), and code 29, which real electric certs in
# the corpus lodge for "room heaters, electric". Collapsing these to one class
# keeps all-electric dwellings comparable to each other; every OTHER fuel keeps
# its raw code as its own bucket (we don't guess a taxonomy for gas/oil/community).
_ELECTRICITY_FUEL_CODES: frozenset[int] = frozenset({29, 30, 31, 32, 33, 34, 35, 38, 40})


def _fuel_class(fuel: Optional[int]) -> str:
    """A coarse main-heating-fuel label for the cohort key. Electricity variants
    collapse to one class; any other code is its own bucket; None is "unmapped"."""
    if fuel is None:
        return "fuel?"
    if fuel in _ELECTRICITY_FUEL_CODES:
        return "electric"
    return f"fuel{fuel}"


def _cohort_key(n: Neighbour) -> Optional[str]:
    """A stable cohort label, or None when the neighbour can't be placed (no
    postcode or no property type — it has no comparable peers to diverge from).

    The main-heating **fuel** is part of the key: electricity scores materially
    lower in SAP than mains gas for identical fabric, so mixing fuels in a cohort
    produces cross-fuel false outliers. Keying on fuel keeps like compared with
    like, so a flagged divergence is a genuine same-fuel anomaly, not just an
    electric dwelling sitting among gas neighbours."""
    if not n.postcode or not n.property_type:
        return None
    form = n.built_form or "?"
    return (
        f"{n.postcode.strip().upper()} · {n.property_type} · {form} "
        f"· {_fuel_class(n.main_fuel_type)}"
    )


def find_divergences(
    neighbours: list[Neighbour], min_gap: float
) -> list[Divergence]:
    cohorts: dict[str, list[Neighbour]] = {}
    for n in neighbours:
        key = _cohort_key(n)
        if key is None or n.effective_sap is None:
            continue
        cohorts.setdefault(key, []).append(n)

    found: list[Divergence] = []
    for key, members in cohorts.items():
        if len(members) < 2:
            continue
        saps = [m.effective_sap for m in members if m.effective_sap is not None]
        cohort_median = median(saps)
        for m in members:
            if m.effective_sap is None:
                continue
            gap = m.effective_sap - cohort_median
            if abs(gap) < min_gap:
                continue
            area = f"{m.floor_area_m2:.0f}m²" if m.floor_area_m2 is not None else "?m²"
            found.append(
                Divergence(
                    property_id=m.property_id,
                    uprn=m.uprn,
                    cohort_key=key,
                    cohort_size=len(members),
                    detail=(
                        f"effective SAP {m.effective_sap:.0f} ({m.effective_band}) "
                        f"vs cohort median {cohort_median:.0f} (Δ{gap:+.0f}, "
                        f"{area}, {len(members)} neighbours)"
                    ),
                )
            )
    found.sort(key=lambda d: (d.cohort_key, d.property_id))
    return found


def _write_reports(divergences: list[Divergence], scanned: int) -> None:
    with open("neighbour_divergence.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["property_id", "uprn", "cohort", "cohort_size", "detail"])
        for d in divergences:
            w.writerow([d.property_id, d.uprn, d.cohort_key, d.cohort_size, d.detail])

    by_cohort: dict[str, list[Divergence]] = {}
    for d in divergences:
        by_cohort.setdefault(d.cohort_key, []).append(d)
    lines = [
        "# Neighbour SAP-divergence audit",
        "",
        f"Scanned **{scanned}** properties · flagged **{len(divergences)}** "
        f"divergent neighbours across **{len(by_cohort)}** cohorts.",
        "",
    ]
    for key in sorted(by_cohort):
        rows = by_cohort[key]
        lines.append(f"## {key} — {len(rows)} divergent")
        lines.append("")
        for d in rows:
            lines.append(f"- property **{d.property_id}** (uprn {d.uprn}): {d.detail}")
        lines.append("")
    with open("neighbour_divergence.md", "w") as f:
        f.write("\n".join(lines))


def main() -> None:
    load_env()
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--portfolio", type=int, required=True, help="portfolio_id")
    parser.add_argument(
        "--min-gap",
        type=float,
        default=_DEFAULT_MIN_GAP,
        help=f"SAP points from cohort median to flag (default {_DEFAULT_MIN_GAP})",
    )
    args = parser.parse_args()

    neighbours = _load(args.portfolio)
    divergences = find_divergences(neighbours, args.min_gap)
    _write_reports(divergences, len(neighbours))

    print(
        f"scanned {len(neighbours)} properties · {len(divergences)} divergent "
        f"neighbours (>= {args.min_gap:.0f} SAP from cohort median)"
    )
    print("wrote neighbour_divergence.md / neighbour_divergence.csv")


if __name__ == "__main__":
    main()