"""Freeze a small, anonymised EPC Prediction fixture for the Tier-1 gate (ADR-0030). Curates a deterministic subset of the local scratch corpus (`/tmp/epc_prediction_corpus`, gitignored) into a committed fixture under `tests/fixtures/epc_prediction/`. Selection keeps postcodes that can actually be scored — at least one SAP 10.2 target plus a second distinct address to predict it from. Every payload is run through `anonymise_payload` first, so the street address + certificate number become opaque tokens and no plaintext address lands in the repo (postcode + component data are open gov data and kept). The committed fixture is the deterministic basis for the ratcheting gate; the large scratch corpus stays local for iteration + the offline battle-test. USAGE ----- PYTHONPATH=. python scripts/build_epc_prediction_fixture.py Source: $EPC_PREDICTION_CORPUS (default /tmp/epc_prediction_corpus). """ from __future__ import annotations import json import os from pathlib import Path from typing import Any from harness.epc_prediction_corpus import anonymise_payload, stable_hash SOURCE = Path(os.environ.get("EPC_PREDICTION_CORPUS", "/tmp/epc_prediction_corpus")) FIXTURE = Path("tests/fixtures/epc_prediction") _SAP_10_2 = "10.2" _MAX_POSTCODES = 15 # keep the committed fixture small _MAX_COHORT = 25 # cap certs per postcode to bound repo size def _load_payloads( postcode: str, certs: list[str] ) -> list[tuple[str, dict[str, Any]]]: """The `(source cert number, payload)` pairs for a postcode — the cert number lives in the index/filename, not the cached payload.""" payloads: list[tuple[str, dict[str, Any]]] = [] for cert in certs: path = SOURCE / postcode / f"{cert}.json" if path.exists(): payloads.append((cert, json.loads(path.read_text()))) return payloads def _qualifies(payloads: list[tuple[str, dict[str, Any]]]) -> bool: """A postcode is usable iff it has ≥1 SAP 10.2 cert (a valid target) and ≥2 distinct addresses (so the target has at least one neighbour to predict it).""" has_target = any( str(p.get("sap_version")) == _SAP_10_2 for _, p in payloads ) addresses = { str(p.get("address_line_1", "")).strip().upper() for _, p in payloads } return has_target and len(addresses) >= 2 def main() -> None: index: dict[str, list[str]] = json.loads( (SOURCE / "_index.json").read_text() ) fixture_index: dict[str, list[str]] = {} total_certs = 0 for postcode, certs in index.items(): if len(fixture_index) >= _MAX_POSTCODES: break payloads = _load_payloads(postcode, certs) if not _qualifies(payloads): continue kept: list[str] = [] for cert, raw in payloads[:_MAX_COHORT]: cert_token = stable_hash("cert", cert) anon = anonymise_payload(raw) out = FIXTURE / postcode / f"{cert_token}.json" out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(anon)) kept.append(cert_token) fixture_index[postcode] = kept total_certs += len(kept) (FIXTURE / "_index.json").parent.mkdir(parents=True, exist_ok=True) (FIXTURE / "_index.json").write_text(json.dumps(fixture_index, indent=2)) print( f"wrote {len(fixture_index)} postcodes / {total_certs} anonymised certs " f"to {FIXTURE}" ) if __name__ == "__main__": main()