"""Step 1 (Durkan portfolio): resolve a UPRN per CSV row via EPC then OS. Input: scripts/lisasrequest/260611_Sample_Seed_Portfolio_Durkan_split_addresses(Split Addresses).csv columns include ``address`` and ``postcode``. Every row carries an address and none carry a UPRN, so there is a single case: * resolve a UPRN from ``address`` + ``postcode`` via the EPC API (relaxed address variants, threshold 0.7), then Ordnance Survey Places as a fallback (threshold 0.6). * not resolvable -> domna_source = "not_found"; uprn/address/score left empty. Writes a NEW CSV = every original column, in order, plus four DOMNA columns: domna_address_found the canonical address EPC/OS returned (matched string) domna_address_uprn the resolved UPRN ("" when unresolved) domna_lexiscore the match score in [0, 1] ("" when unresolved) domna_source epc / ordnance_survey / not_found This is the human-review file; step 2 (resolve_uprns_for_finaliser) reshapes it into the finaliser columns without re-hitting the APIs. python -m scripts.lisasrequest.fill_domna_address python -m scripts.lisasrequest.fill_domna_address --limit 20 # smoke test Resolution reuses the relaxed matchers from scripts.fill_domna_addresses. Keys come from backend/.env (OPEN_EPC_API_TOKEN, ORDNANCE_SURVEY_API_KEY). Run from the worktree root (import trap). """ from __future__ import annotations import argparse import csv import sys from pathlib import Path from typing import Optional import pandas as pd _REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(_REPO_ROOT)) # worktree root first — avoid the import trap from scripts.fill_domna_addresses import ( # noqa: E402 Hit, resolve_epc_relaxed, resolve_os_relaxed, ) from scripts.resolve_uprns_for_finaliser import clean_postcode, load_keys # noqa: E402 ADDRESS_COL = "address" POSTCODE_COL = "postcode" FOUND_ADDRESS_COL = "domna_address_found" FOUND_UPRN_COL = "domna_address_uprn" LEXISCORE_COL = "domna_lexiscore" SOURCE_COL = "domna_source" NOT_FOUND = "not_found" _RESULT_COLS = (FOUND_ADDRESS_COL, FOUND_UPRN_COL, LEXISCORE_COL, SOURCE_COL) _CSV_NAME = "260611_Sample_Seed_Portfolio_Durkan_split_addresses(Split Addresses).csv" _DEFAULT_IN = _REPO_ROOT / "scripts" / "lisasrequest" / _CSV_NAME _DEFAULT_OUT = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_domna_filled.csv" def read_rows(path: Path) -> tuple[list[dict[str, str]], list[str]]: """Read a CSV into (rows, fieldnames), preserving column order.""" with path.open(newline="", encoding="utf-8-sig") as fh: reader = csv.DictReader(fh) fieldnames = list(reader.fieldnames or []) rows = [dict(row) for row in reader] return rows, fieldnames def resolve_one( address: str, postcode_raw: str, *, epc_token: Optional[str], os_api_key: Optional[str], epc_cache: dict[str, pd.DataFrame], os_cache: dict[str, pd.DataFrame], ) -> Optional[Hit]: """Resolve one row's UPRN: EPC (relaxed) first, then OS Places fallback.""" postcode_clean = clean_postcode(postcode_raw) if not address or not postcode_clean: return None hit: Optional[Hit] = None if epc_token: try: hit = resolve_epc_relaxed(address, postcode_clean, epc_cache) except Exception as exc: print(f" EPC failed {address!r} / {postcode_clean}: {exc}") if hit is None and os_api_key: try: hit = resolve_os_relaxed(address, postcode_clean, os_api_key, os_cache) except Exception as exc: print(f" OS failed {address!r} / {postcode_clean}: {exc}") return hit def fill( rows: list[dict[str, str]], *, epc_token: Optional[str], os_api_key: Optional[str], ) -> tuple[int, int, int]: """Fill the DOMNA columns on each row in place. Returns (epc_hits, os_hits, not_found) counts. """ epc_cache: dict[str, pd.DataFrame] = {} os_cache: dict[str, pd.DataFrame] = {} epc_hits = os_hits = not_found = 0 total = len(rows) for n, row in enumerate(rows, start=1): address = str(row.get(ADDRESS_COL, "") or "").strip() postcode_raw = str(row.get(POSTCODE_COL, "") or "").strip() hit = resolve_one( address, postcode_raw, epc_token=epc_token, os_api_key=os_api_key, epc_cache=epc_cache, os_cache=os_cache, ) if hit is None: row[FOUND_ADDRESS_COL] = "" row[FOUND_UPRN_COL] = "" row[LEXISCORE_COL] = "" row[SOURCE_COL] = NOT_FOUND not_found += 1 else: uprn, matched, score, source = hit row[FOUND_ADDRESS_COL] = matched row[FOUND_UPRN_COL] = uprn row[LEXISCORE_COL] = str(round(score, 4)) row[SOURCE_COL] = source if source == "epc": epc_hits += 1 else: os_hits += 1 print( f"[{n}/{total}] {address!r} -> " f"{row[FOUND_UPRN_COL] or '(no match)'} ({row[SOURCE_COL]})" ) return epc_hits, os_hits, not_found def write_rows(rows: list[dict[str, str]], path: Path, fieldnames: list[str]) -> None: """Write rows to CSV, preserving input columns and appending DOMNA columns.""" out_fields = list(fieldnames) for col in _RESULT_COLS: if col not in out_fields: out_fields.append(col) with path.open("w", newline="", encoding="utf-8") as fh: writer = csv.DictWriter(fh, fieldnames=out_fields, extrasaction="ignore") writer.writeheader() writer.writerows(rows) def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--in", dest="inp", type=Path, default=_DEFAULT_IN) parser.add_argument("--out", type=Path, default=_DEFAULT_OUT) parser.add_argument("--limit", type=int, default=None, help="process first N rows") return parser.parse_args() def main() -> int: args = _parse_args() epc_token, os_api_key = load_keys() if not epc_token: print("OPEN_EPC_API_TOKEN not set (backend/.env) — EPC resolution disabled") if not os_api_key: print("ORDNANCE_SURVEY_API_KEY not set (backend/.env) — OS fallback disabled") rows, fieldnames = read_rows(args.inp) if args.limit is not None: rows = rows[: args.limit] print(f"Loaded {len(rows)} rows from {args.inp}") epc_hits, os_hits, not_found = fill( rows, epc_token=epc_token, os_api_key=os_api_key ) write_rows(rows, args.out, fieldnames) resolved = epc_hits + os_hits print( f"\nResolved {resolved}/{len(rows)} " f"(epc={epc_hits}, ordnance_survey={os_hits}); {not_found} not found." ) print(f"Wrote filled CSV -> {args.out}") return 0 if __name__ == "__main__": sys.exit(main())