Model/scripts/build_epc_prediction_fixture.py

"""Freeze a small, anonymised EPC Prediction fixture for the Tier-1 gate (ADR-0030).

Curates a deterministic subset of the local scratch corpus
(`/tmp/epc_prediction_corpus`, gitignored) into a committed fixture under
`tests/fixtures/epc_prediction/`. Selection keeps postcodes that can actually be
scored — at least one SAP 10.2 target plus a second distinct address to predict
it from. Every payload is run through `anonymise_payload` first, so the street
address + certificate number become opaque tokens and no plaintext address lands
in the repo (postcode + component data are open gov data and kept).

The committed fixture is the deterministic basis for the ratcheting gate; the
large scratch corpus stays local for iteration + the offline battle-test.

USAGE
-----
    PYTHONPATH=. python scripts/build_epc_prediction_fixture.py

Source: $EPC_PREDICTION_CORPUS (default /tmp/epc_prediction_corpus).
"""

from __future__ import annotations

import json
import os
from pathlib import Path
from typing import Any

from harness.epc_prediction_corpus import anonymise_payload, stable_hash

SOURCE = Path(os.environ.get("EPC_PREDICTION_CORPUS", "/tmp/epc_prediction_corpus"))
FIXTURE = Path("tests/fixtures/epc_prediction")

_SAP_10_2 = "10.2"
_MAX_POSTCODES = 15       # keep the committed fixture small
_MAX_COHORT = 25          # cap certs per postcode to bound repo size


def _load_payloads(
    postcode: str, certs: list[str]
) -> list[tuple[str, dict[str, Any]]]:
    """The `(source cert number, payload)` pairs for a postcode — the cert
    number lives in the index/filename, not the cached payload."""
    payloads: list[tuple[str, dict[str, Any]]] = []
    for cert in certs:
        path = SOURCE / postcode / f"{cert}.json"
        if path.exists():
            payloads.append((cert, json.loads(path.read_text())))
    return payloads


def _qualifies(payloads: list[tuple[str, dict[str, Any]]]) -> bool:
    """A postcode is usable iff it has ≥1 SAP 10.2 cert (a valid target) and ≥2
    distinct addresses (so the target has at least one neighbour to predict it)."""
    has_target = any(
        str(p.get("sap_version")) == _SAP_10_2 for _, p in payloads
    )
    addresses = {
        str(p.get("address_line_1", "")).strip().upper() for _, p in payloads
    }
    return has_target and len(addresses) >= 2


def main() -> None:
    index: dict[str, list[str]] = json.loads(
        (SOURCE / "_index.json").read_text()
    )
    fixture_index: dict[str, list[str]] = {}
    kept_uprns: set[str] = set()
    total_certs = 0
    for postcode, certs in index.items():
        if len(fixture_index) >= _MAX_POSTCODES:
            break
        payloads = _load_payloads(postcode, certs)
        if not _qualifies(payloads):
            continue
        kept: list[str] = []
        for cert, raw in payloads[:_MAX_COHORT]:
            cert_token = stable_hash("cert", cert)
            anon = anonymise_payload(raw)
            out = FIXTURE / postcode / f"{cert_token}.json"
            out.parent.mkdir(parents=True, exist_ok=True)
            out.write_text(json.dumps(anon))
            kept.append(cert_token)
            uprn = raw.get("uprn")
            if uprn is not None:
                kept_uprns.add(str(int(uprn)))
        fixture_index[postcode] = kept
        total_certs += len(kept)
    (FIXTURE / "_index.json").parent.mkdir(parents=True, exist_ok=True)
    (FIXTURE / "_index.json").write_text(json.dumps(fixture_index, indent=2))
    _write_coordinates(kept_uprns)
    print(
        f"wrote {len(fixture_index)} postcodes / {total_certs} anonymised certs "
        f"to {FIXTURE}"
    )


def _write_coordinates(kept_uprns: set[str]) -> None:
    """Carry the geo-proximity coordinates for the kept UPRNs into the committed
    fixture (subset of the corpus `_coordinates.json`), so the gate exercises
    geo-weighting without S3. Skipped when the corpus has no coordinates sidecar.
    Coordinates are OS OpenData (OGL) and add no identifiability beyond the UPRN
    already kept in the fixture."""
    source = SOURCE / "_coordinates.json"
    if not source.exists():
        return
    corpus_coords: dict[str, list[float]] = json.loads(source.read_text())
    fixture_coords = {
        uprn: corpus_coords[uprn]
        for uprn in kept_uprns
        if uprn in corpus_coords
    }
    (FIXTURE / "_coordinates.json").write_text(json.dumps(fixture_coords))


if __name__ == "__main__":
    main()