Model/scripts/build_expired_pairs_corpus.py

"""Freeze a subsample of harness pairs into the committed integration fixture.

Turns the pairs harness's live evidence into a deterministic, offline
regression gate (the ADR-0030 corpus pattern): anonymised RAW API payloads
loaded through ``EpcPropertyDataMapper``, so the gate keeps exercising the
mapper and survives domain-dataclass changes. Layout, extending the
``tests/fixtures/epc_prediction`` conventions:

    ONE json file (a thousand per-cert files would drown a PR diff):
      pairs:   [{postcode, uprn, actual, historic: {...}}]
      cohorts: {postcode: {token: anonymised payload}}
      actuals: {token: anonymised payload}   (the lodged SAP-10.2 certs)

Reads the raw-JSON disk cache (scripts/epc_disk_cache.py) for everything the
API served, and the historic S3 backup for the pre-2012 records. Pairs are
subsampled deterministically (sorted, strided) from the harness telemetry.

Usage:
    python scripts/build_expired_pairs_corpus.py \
        --telemetry pairs_telemetry.jsonl --cache-dir .epc_cache \
        --out tests/fixtures/expired_prediction_pairs.json --sample 30
"""

from __future__ import annotations

import argparse
import dataclasses
import json
import sys
from pathlib import Path
from typing import Any, Optional

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from datatypes.epc.domain.historic_epc import HistoricEpc  # noqa: E402
from domain.postcode import Postcode  # noqa: E402
from harness.epc_prediction_corpus import anonymise_payload, stable_hash  # noqa: E402

# Free-text fields blanked on the frozen historic record; the postcode is kept
# (coarse open data, the shard key) and the address becomes a stable token so
# nothing joins back to a household.
_HISTORIC_PII_BLANK = ("address1", "address2", "address3", "posttown")


def anonymise_historic(record: HistoricEpc) -> dict[str, str]:
    row = dataclasses.asdict(record)
    for field in _HISTORIC_PII_BLANK:
        row[field] = ""
    row["address"] = stable_hash("addr", record.address) if record.address else ""
    row["lmk_key"] = stable_hash("lmk", record.lmk_key) if record.lmk_key else ""
    return row


def _read_cache(cache_dir: Path, key: str) -> Optional[Any]:
    path = cache_dir / f"{key}.json"
    return json.loads(path.read_text()) if path.exists() else None


def subsample(rows: list[dict[str, Any]], count: int) -> list[dict[str, Any]]:
    """A deterministic spread across the telemetry: sort, stride."""
    ordered = sorted(rows, key=lambda r: (str(r["postcode"]), str(r["uprn"])))
    if len(ordered) <= count:
        return ordered
    stride = len(ordered) // count
    return ordered[::stride][:count]


def main() -> None:  # pragma: no cover - IO composition
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--telemetry", type=Path, required=True)
    parser.add_argument("--cache-dir", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--sample", type=int, default=30)
    args = parser.parse_args()

    from repositories.historic_epc.historic_epc_resolver import HistoricEpcResolver
    from repositories.historic_epc.historic_epc_s3_repository import (
        HistoricEpcS3Repository,
    )

    resolver = HistoricEpcResolver(HistoricEpcS3Repository.with_default_s3_client())

    telemetry = [json.loads(line) for line in args.telemetry.read_text().splitlines()]
    chosen = subsample(telemetry, args.sample)

    cohorts: dict[str, dict[str, Any]] = {}
    actuals: dict[str, Any] = {}
    pairs: list[dict[str, Any]] = []
    for row in chosen:
        postcode, uprn = str(row["postcode"]), str(row["uprn"])
        historic = resolver.record_for_uprn(uprn, postcode)
        if historic is None:
            print(f"{postcode} {uprn}: historic record gone — skipped", file=sys.stderr)
            continue
        search = _read_cache(args.cache_dir, f"search_uprn_{uprn}")
        if not search:
            print(f"{postcode} {uprn}: uprn search not cached — skipped", file=sys.stderr)
            continue
        latest = max(search, key=lambda r: str(r["registration_date"]))
        actual_raw = _read_cache(args.cache_dir, f"cert_{latest['certificate_number']}")
        cohort_search = _read_cache(args.cache_dir, f"search_pc_{postcode}")
        if actual_raw is None or cohort_search is None:
            print(f"{postcode} {uprn}: cert/cohort not cached — skipped", file=sys.stderr)
            continue

        if postcode not in cohorts:
            payloads: dict[str, Any] = {}
            for result in cohort_search:
                raw = _read_cache(
                    args.cache_dir, f"cert_{result['certificate_number']}"
                )
                if raw is None:
                    continue
                token = stable_hash("cert", str(result["certificate_number"]))
                payloads[token] = anonymise_payload(raw)
            cohorts[postcode] = payloads

        actual_token = stable_hash("cert", str(latest["certificate_number"]))
        actuals[actual_token] = anonymise_payload(actual_raw)
        pairs.append(
            {
                "postcode": str(Postcode(postcode)),
                "uprn": uprn,
                "actual": actual_token,
                "historic": anonymise_historic(historic),
            }
        )
        print(f"{postcode} {uprn}: frozen", file=sys.stderr)

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(
        json.dumps(
            {"pairs": pairs, "cohorts": cohorts, "actuals": actuals}, indent=1
        )
    )
    print(f"{len(pairs)} pairs frozen across {len(cohorts)} postcodes -> {args.out}")


if __name__ == "__main__":
    main()