"""Freeze a small, anonymised EPC Prediction fixture for the Tier-1 gate (ADR-0030). Curates a deterministic subset of the local scratch corpus (`/tmp/epc_prediction_corpus`, gitignored) into a committed fixture under `tests/fixtures/epc_prediction/`. Selection keeps postcodes that can actually be scored — at least one SAP 10.2 target plus a second distinct address to predict it from. Every payload is run through `anonymise_payload` first, so the street address + certificate number become opaque tokens and no plaintext address lands in the repo (postcode + component data are open gov data and kept). The committed fixture is the deterministic basis for the ratcheting gate; the large scratch corpus stays local for iteration + the offline battle-test. USAGE ----- PYTHONPATH=. python scripts/build_epc_prediction_fixture.py Source: $EPC_PREDICTION_CORPUS (default /tmp/epc_prediction_corpus). """ from __future__ import annotations import json import os from pathlib import Path from typing import Any from harness.epc_prediction_corpus import anonymise_payload, stable_hash SOURCE = Path(os.environ.get("EPC_PREDICTION_CORPUS", "/tmp/epc_prediction_corpus")) FIXTURE = Path("tests/fixtures/epc_prediction") _SAP_10_2 = "10.2" _MAX_POSTCODES = 15 # keep the committed fixture small _MAX_COHORT = 25 # cap certs per postcode to bound repo size def _load_payloads( postcode: str, certs: list[str] ) -> list[tuple[str, dict[str, Any]]]: """The `(source cert number, payload)` pairs for a postcode — the cert number lives in the index/filename, not the cached payload.""" payloads: list[tuple[str, dict[str, Any]]] = [] for cert in certs: path = SOURCE / postcode / f"{cert}.json" if path.exists(): payloads.append((cert, json.loads(path.read_text()))) return payloads def _qualifies(payloads: list[tuple[str, dict[str, Any]]]) -> bool: """A postcode is usable iff it has ≥1 SAP 10.2 cert (a valid target) and ≥2 distinct addresses (so the target has at least one neighbour to predict it).""" has_target = any( str(p.get("sap_version")) == _SAP_10_2 for _, p in payloads ) addresses = { str(p.get("address_line_1", "")).strip().upper() for _, p in payloads } return has_target and len(addresses) >= 2 def main() -> None: index: dict[str, list[str]] = json.loads( (SOURCE / "_index.json").read_text() ) fixture_index: dict[str, list[str]] = {} kept_uprns: set[str] = set() total_certs = 0 for postcode, certs in index.items(): if len(fixture_index) >= _MAX_POSTCODES: break payloads = _load_payloads(postcode, certs) if not _qualifies(payloads): continue kept: list[str] = [] for cert, raw in payloads[:_MAX_COHORT]: cert_token = stable_hash("cert", cert) anon = anonymise_payload(raw) out = FIXTURE / postcode / f"{cert_token}.json" out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(anon)) kept.append(cert_token) uprn = raw.get("uprn") if uprn is not None: kept_uprns.add(str(int(uprn))) fixture_index[postcode] = kept total_certs += len(kept) (FIXTURE / "_index.json").parent.mkdir(parents=True, exist_ok=True) (FIXTURE / "_index.json").write_text(json.dumps(fixture_index, indent=2)) _write_coordinates(kept_uprns) print( f"wrote {len(fixture_index)} postcodes / {total_certs} anonymised certs " f"to {FIXTURE}" ) def _write_coordinates(kept_uprns: set[str]) -> None: """Carry the geo-proximity coordinates for the kept UPRNs into the committed fixture (subset of the corpus `_coordinates.json`), so the gate exercises geo-weighting without S3. Skipped when the corpus has no coordinates sidecar. Coordinates are OS OpenData (OGL) and add no identifiability beyond the UPRN already kept in the fixture.""" source = SOURCE / "_coordinates.json" if not source.exists(): return corpus_coords: dict[str, list[float]] = json.loads(source.read_text()) fixture_coords = { uprn: corpus_coords[uprn] for uprn in kept_uprns if uprn in corpus_coords } (FIXTURE / "_coordinates.json").write_text(json.dumps(fixture_coords)) if __name__ == "__main__": main()