Model/harness/epc_prediction_corpus.py

"""Load a postcode-clustered EPC corpus into ComparableProperty cohorts (ADR-0030).

The IO half of the EPC Prediction validation: read each postcode's cached cert
payloads, map them through `EpcPropertyDataMapper.from_api_response`, and build
`ComparableProperty`s carrying the register metadata (address + registration date) the
leave-one-out scorer needs to dedupe re-lodgements and hold out a whole address.
A cert the mapper rejects (unsupported schema, malformed) is skipped, never fatal.

Shared by the committed-fixture gate, the local validation script, and the
offline national battle-test — the corpus directory differs, the loading does
not. Layout: `<dir>/<POSTCODE>/<cert>.json` + `<dir>/_index.json`.
"""

from __future__ import annotations

import hashlib
import json
from datetime import date
from pathlib import Path
from typing import Any, Optional

from datatypes.epc.domain.mapper import EpcPropertyDataMapper
from domain.epc_prediction.comparable_properties import ComparableProperty
from domain.geospatial.coordinates import Coordinates

# Identifying free-text fields blanked when freezing a payload into the committed
# fixture (postcode is kept — it is coarse open data and the cohort key).
_PII_BLANK_FIELDS = ("address_line_2", "address_line_3", "post_town")


def load_corpus(corpus_dir: Path) -> list[list[ComparableProperty]]:
    """Load every postcode cohort under `corpus_dir`. Returns one list of
    Comparables per postcode (the unit the leave-one-out scorer iterates)."""
    index_path = corpus_dir / "_index.json"
    if not index_path.exists():
        raise FileNotFoundError(
            f"no corpus index at {index_path} — run a corpus fetch first"
        )
    index: dict[str, list[str]] = json.loads(index_path.read_text())
    coordinates = load_coordinates(corpus_dir)
    return [
        _load_cohort(corpus_dir, postcode, certs, coordinates)
        for postcode, certs in index.items()
    ]


def _load_cohort(
    corpus_dir: Path,
    postcode: str,
    certs: list[str],
    coordinates: dict[int, Coordinates],
) -> list[ComparableProperty]:
    cohort: list[ComparableProperty] = []
    for cert in certs:
        path = corpus_dir / postcode / f"{cert}.json"
        if not path.exists():
            continue
        raw: dict[str, Any] = json.loads(path.read_text())
        comparable = comparable_from_payload(cert, raw, coordinates)
        if comparable is not None:
            cohort.append(comparable)
    return cohort


def comparable_from_payload(
    cert: str,
    raw: dict[str, Any],
    coordinates: dict[int, Coordinates],
) -> Optional[ComparableProperty]:
    """One frozen cert payload -> a ComparableProperty, or None when the mapper
    can't take it (a bad cert must never abort a corpus load). Shared by the
    per-file corpus above and the single-file expired-pairs corpus."""
    try:
        epc = EpcPropertyDataMapper.from_api_response(raw)
    except Exception:  # noqa: BLE001
        return None
    uprn = _uprn(raw)
    return ComparableProperty(
        epc=epc,
        certificate_number=cert,
        address=_address(raw),
        registration_date=_registration_date(raw),
        coordinates=coordinates.get(uprn) if uprn is not None else None,
    )


def load_coordinates(corpus_dir: Path) -> dict[int, Coordinates]:
    """The optional `_coordinates.json` sidecar (`{uprn: [lon, lat]}`), resolved
    from the OS Open-UPRN data by `fetch_corpus_coordinates.py`. Absent for a
    corpus without geo data — geo-weighting then simply stays off."""
    path = corpus_dir / "_coordinates.json"
    if not path.exists():
        return {}
    raw: dict[str, list[float]] = json.loads(path.read_text())
    return {
        int(uprn): Coordinates(longitude=lon_lat[0], latitude=lon_lat[1])
        for uprn, lon_lat in raw.items()
    }


def _uprn(raw: dict[str, Any]) -> Optional[int]:
    value = raw.get("uprn")
    return int(value) if value is not None else None


def stable_hash(prefix: str, value: str) -> str:
    """A short, deterministic, one-way token for a free-text identifier. Stable
    across re-lodgements of the same address (normalised first), so dedup still
    collapses them — but the plaintext address never lands in the repo."""
    digest = hashlib.sha1(value.strip().upper().encode()).hexdigest()[:12]
    return f"{prefix}-{digest}"


def anonymise_payload(raw: dict[str, Any]) -> dict[str, Any]:
    """De-identify a cert payload for the committed fixture: hash the street
    address (`address_line_1`) and certificate number into stable tokens, blank
    the other free-text address lines, and keep everything else — postcode,
    registration date, SAP version, lodged figures, and all component fields —
    untouched (gov data is OGL; only the direct identifiers are removed)."""
    out = dict(raw)
    address = raw.get("address_line_1")
    if address:
        out["address_line_1"] = stable_hash("addr", str(address))
    cert = raw.get("certificate_number")
    if cert:
        out["certificate_number"] = stable_hash("cert", str(cert))
    for blank_field in _PII_BLANK_FIELDS:
        if blank_field in out:
            out[blank_field] = ""
    return out


def _address(raw: dict[str, Any]) -> Optional[str]:
    value = raw.get("address_line_1")
    return str(value).strip().upper() if value else None


def _registration_date(raw: dict[str, Any]) -> Optional[date]:
    value = raw.get("registration_date")
    return date.fromisoformat(str(value)) if value else None