Model/scripts/lisasrequest/compare_to_ara.py

"""Compare our step-1 UPRN resolution against the old "Ara output" data.

The Ara data lives in scripts/lisasrequest/Durkan data.xlsx, sheet "Ara output",
and carries UPRNs from our previous dataset. It is NOT treated as ground truth —
this just lines it up against what we found / didn't find so a human can eyeball
the differences. (We read the xlsx, not the CSV export: the CSV mangled half the
UPRNs to Excel scientific notation, e.g. ``1.00023E+11``; the xlsx keeps them
intact, so every comparison below is exact.)

Join key is (postcode, leading number, first street word), since the UPRN is the
thing under comparison and Ara's address strings differ from the landlord input.

Each of our rows lands in one comparison bucket:
    match            both found a UPRN and they are equal.
    differ           both found a UPRN and they differ.
    we_only          we resolved a UPRN, Ara had none for this address.
    ara_only         we did NOT resolve, but Ara had a UPRN  <- recovery candidates.
    both_missing     neither resolved a UPRN.
    no_ara_record    the Ara sheet had no row matching this address at all.

    python -m scripts.lisasrequest.compare_to_ara
"""

from __future__ import annotations

import argparse
import csv
import re
import sys
from collections import Counter, OrderedDict
from pathlib import Path
from typing import Optional

import pandas as pd

_REPO_ROOT = Path(__file__).resolve().parents[2]

ADDRESS_COL = "address"
POSTCODE_COL = "postcode"
OUR_UPRN_COL = "domna_address_uprn"
OUR_SOURCE_COL = "domna_source"

ARA_UPRN_COL = "EPC_B.uprn"
ARA_ADDRESS_COL = "EPC_B.address"
ARA_POSTCODE_COL = "EPC_B.postcode"
ARA_SHEET = "Ara output"

_OUR_IN = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_domna_filled.csv"
_ARA_IN = _REPO_ROOT / "scripts" / "lisasrequest" / "Durkan data.xlsx"
_DEFAULT_OUT = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_vs_ara.csv"

Key = tuple[str, str, str]


def norm_key(address: str, postcode: str) -> Key:
    """(postcode-no-space, leading number, first street word) — the join key."""
    pc = postcode.upper().replace(" ", "")
    upper = address.upper()
    nums = re.findall(r"\d+[A-Z]?", upper)
    words = [w for w in re.findall(r"[A-Z]+", upper) if w != "FLAT"]
    return (pc, nums[0] if nums else "", words[0] if words else "")


def load_ara(path: Path) -> tuple[dict[Key, dict[str, str]], int]:
    """Index the Ara-output xlsx sheet by join key (first row wins).

    Returns (index, duplicates). Read as strings so UPRNs keep their full value.
    """
    df = pd.read_excel(path, sheet_name=ARA_SHEET, dtype=str)
    rows: list[dict[str, str]] = df.fillna("").to_dict(orient="records")
    index: dict[Key, dict[str, str]] = OrderedDict()
    dupes = 0
    for row in rows:
        address = str(row.get(ARA_ADDRESS_COL) or "").strip()
        postcode = str(row.get(ARA_POSTCODE_COL) or row.get(POSTCODE_COL) or "").strip()
        if not address:
            continue
        key = norm_key(address, postcode)
        if key in index:
            dupes += 1
            continue
        index[key] = row
    return index, dupes


def classify(
    our_uprn: str, our_found: bool, ara: Optional[dict[str, str]]
) -> tuple[str, str, str]:
    """Return (comparison, ara_uprn, ara_address) for one of our rows."""
    if ara is None:
        return ("no_ara_record", "", "")
    ara_uprn = (ara.get(ARA_UPRN_COL) or "").strip()
    ara_address = (ara.get(ARA_ADDRESS_COL) or "").strip()
    ara_found = bool(ara_uprn)

    if our_found and ara_found:
        comparison = "match" if our_uprn == ara_uprn else "differ"
    elif our_found and not ara_found:
        comparison = "we_only"
    elif not our_found and ara_found:
        comparison = "ara_only"
    else:
        comparison = "both_missing"
    return (comparison, ara_uprn, ara_address)


def compare(
    our_rows: list[dict[str, str]], ara_index: dict[Key, dict[str, str]]
) -> list[dict[str, str]]:
    out: list[dict[str, str]] = []
    for row in our_rows:
        address = (row.get(ADDRESS_COL) or "").strip()
        postcode = (row.get(POSTCODE_COL) or "").strip()
        our_uprn = (row.get(OUR_UPRN_COL) or "").strip()
        our_source = (row.get(OUR_SOURCE_COL) or "").strip()
        our_found = bool(our_uprn) and our_source != "not_found"

        ara = ara_index.get(norm_key(address, postcode))
        comparison, ara_uprn, ara_address = classify(our_uprn, our_found, ara)
        out.append(
            {
                "address": address,
                "postcode": postcode,
                "our_uprn": our_uprn,
                "our_source": our_source,
                "ara_uprn": ara_uprn,
                "ara_address": ara_address,
                "comparison": comparison,
            }
        )
    return out


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--ours", type=Path, default=_OUR_IN)
    parser.add_argument("--ara", type=Path, default=_ARA_IN)
    parser.add_argument("--out", type=Path, default=_DEFAULT_OUT)
    args = parser.parse_args()

    with args.ours.open(newline="", encoding="utf-8-sig") as fh:
        our_rows = [dict(r) for r in csv.DictReader(fh)]
    ara_index, dupes = load_ara(args.ara)
    print(f"Loaded {len(our_rows)} of our rows; {len(ara_index)} Ara keys "
          f"({dupes} duplicate Ara rows ignored).")

    result = compare(our_rows, ara_index)
    fieldnames = list(result[0].keys())
    with args.out.open("w", newline="", encoding="utf-8") as fh:
        writer = csv.DictWriter(fh, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(result)

    counts = Counter(r["comparison"] for r in result)
    print(f"\nComparison of {len(result)} rows -> {args.out}")
    for name in (
        "match",
        "differ",
        "we_only",
        "ara_only",
        "both_missing",
        "no_ara_record",
    ):
        print(f"  {name}: {counts.get(name, 0)}")
    return 0


if __name__ == "__main__":
    sys.exit(main())