Model/scripts/lisasrequest/fill_domna_address.py

"""Step 1 (Durkan portfolio): resolve a UPRN per CSV row via EPC then OS.

Input: scripts/lisasrequest/260611_Sample_Seed_Portfolio_Durkan_split_addresses(Split Addresses).csv
    columns include ``address`` and ``postcode``.

Every row carries an address and none carry a UPRN, so there is a single case:

  * resolve a UPRN from ``address`` + ``postcode`` via the EPC API (relaxed
    address variants, threshold 0.7), then Ordnance Survey Places as a fallback
    (threshold 0.6).
  * not resolvable -> domna_source = "not_found"; uprn/address/score left empty.

Writes a NEW CSV = every original column, in order, plus four DOMNA columns:

    domna_address_found   the canonical address EPC/OS returned (matched string)
    domna_address_uprn    the resolved UPRN ("" when unresolved)
    domna_lexiscore       the match score in [0, 1] ("" when unresolved)
    domna_source          epc / ordnance_survey / not_found

This is the human-review file; step 2 (resolve_uprns_for_finaliser) reshapes it
into the finaliser columns without re-hitting the APIs.

    python -m scripts.lisasrequest.fill_domna_address
    python -m scripts.lisasrequest.fill_domna_address --limit 20   # smoke test

Resolution reuses the relaxed matchers from scripts.fill_domna_addresses. Keys
come from backend/.env (OPEN_EPC_API_TOKEN, ORDNANCE_SURVEY_API_KEY). Run from
the worktree root (import trap).
"""

from __future__ import annotations

import argparse
import csv
import sys
from pathlib import Path
from typing import Optional

import pandas as pd

_REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(_REPO_ROOT))  # worktree root first — avoid the import trap

from scripts.fill_domna_addresses import (  # noqa: E402
    Hit,
    resolve_epc_relaxed,
    resolve_os_relaxed,
)
from scripts.resolve_uprns_for_finaliser import clean_postcode, load_keys  # noqa: E402

ADDRESS_COL = "address"
POSTCODE_COL = "postcode"
FOUND_ADDRESS_COL = "domna_address_found"
FOUND_UPRN_COL = "domna_address_uprn"
LEXISCORE_COL = "domna_lexiscore"
SOURCE_COL = "domna_source"
NOT_FOUND = "not_found"
_RESULT_COLS = (FOUND_ADDRESS_COL, FOUND_UPRN_COL, LEXISCORE_COL, SOURCE_COL)

_CSV_NAME = "260611_Sample_Seed_Portfolio_Durkan_split_addresses(Split Addresses).csv"
_DEFAULT_IN = _REPO_ROOT / "scripts" / "lisasrequest" / _CSV_NAME
_DEFAULT_OUT = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_domna_filled.csv"


def read_rows(path: Path) -> tuple[list[dict[str, str]], list[str]]:
    """Read a CSV into (rows, fieldnames), preserving column order."""
    with path.open(newline="", encoding="utf-8-sig") as fh:
        reader = csv.DictReader(fh)
        fieldnames = list(reader.fieldnames or [])
        rows = [dict(row) for row in reader]
    return rows, fieldnames


def resolve_one(
    address: str,
    postcode_raw: str,
    *,
    epc_token: Optional[str],
    os_api_key: Optional[str],
    epc_cache: dict[str, pd.DataFrame],
    os_cache: dict[str, pd.DataFrame],
) -> Optional[Hit]:
    """Resolve one row's UPRN: EPC (relaxed) first, then OS Places fallback."""
    postcode_clean = clean_postcode(postcode_raw)
    if not address or not postcode_clean:
        return None

    hit: Optional[Hit] = None
    if epc_token:
        try:
            hit = resolve_epc_relaxed(address, postcode_clean, epc_cache)
        except Exception as exc:
            print(f"  EPC failed {address!r} / {postcode_clean}: {exc}")
    if hit is None and os_api_key:
        try:
            hit = resolve_os_relaxed(address, postcode_clean, os_api_key, os_cache)
        except Exception as exc:
            print(f"  OS failed {address!r} / {postcode_clean}: {exc}")
    return hit


def fill(
    rows: list[dict[str, str]],
    *,
    epc_token: Optional[str],
    os_api_key: Optional[str],
) -> tuple[int, int, int]:
    """Fill the DOMNA columns on each row in place.

    Returns (epc_hits, os_hits, not_found) counts.
    """
    epc_cache: dict[str, pd.DataFrame] = {}
    os_cache: dict[str, pd.DataFrame] = {}
    epc_hits = os_hits = not_found = 0
    total = len(rows)

    for n, row in enumerate(rows, start=1):
        address = str(row.get(ADDRESS_COL, "") or "").strip()
        postcode_raw = str(row.get(POSTCODE_COL, "") or "").strip()
        hit = resolve_one(
            address,
            postcode_raw,
            epc_token=epc_token,
            os_api_key=os_api_key,
            epc_cache=epc_cache,
            os_cache=os_cache,
        )
        if hit is None:
            row[FOUND_ADDRESS_COL] = ""
            row[FOUND_UPRN_COL] = ""
            row[LEXISCORE_COL] = ""
            row[SOURCE_COL] = NOT_FOUND
            not_found += 1
        else:
            uprn, matched, score, source = hit
            row[FOUND_ADDRESS_COL] = matched
            row[FOUND_UPRN_COL] = uprn
            row[LEXISCORE_COL] = str(round(score, 4))
            row[SOURCE_COL] = source
            if source == "epc":
                epc_hits += 1
            else:
                os_hits += 1
        print(
            f"[{n}/{total}] {address!r} -> "
            f"{row[FOUND_UPRN_COL] or '(no match)'} ({row[SOURCE_COL]})"
        )

    return epc_hits, os_hits, not_found


def write_rows(rows: list[dict[str, str]], path: Path, fieldnames: list[str]) -> None:
    """Write rows to CSV, preserving input columns and appending DOMNA columns."""
    out_fields = list(fieldnames)
    for col in _RESULT_COLS:
        if col not in out_fields:
            out_fields.append(col)
    with path.open("w", newline="", encoding="utf-8") as fh:
        writer = csv.DictWriter(fh, fieldnames=out_fields, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(rows)


def _parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--in", dest="inp", type=Path, default=_DEFAULT_IN)
    parser.add_argument("--out", type=Path, default=_DEFAULT_OUT)
    parser.add_argument("--limit", type=int, default=None, help="process first N rows")
    return parser.parse_args()


def main() -> int:
    args = _parse_args()
    epc_token, os_api_key = load_keys()
    if not epc_token:
        print("OPEN_EPC_API_TOKEN not set (backend/.env) — EPC resolution disabled")
    if not os_api_key:
        print("ORDNANCE_SURVEY_API_KEY not set (backend/.env) — OS fallback disabled")

    rows, fieldnames = read_rows(args.inp)
    if args.limit is not None:
        rows = rows[: args.limit]
    print(f"Loaded {len(rows)} rows from {args.inp}")

    epc_hits, os_hits, not_found = fill(
        rows, epc_token=epc_token, os_api_key=os_api_key
    )

    write_rows(rows, args.out, fieldnames)
    resolved = epc_hits + os_hits
    print(
        f"\nResolved {resolved}/{len(rows)} "
        f"(epc={epc_hits}, ordnance_survey={os_hits}); {not_found} not found."
    )
    print(f"Wrote filled CSV -> {args.out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())