Model/scripts/lisasrequest/durkan_805_schema_check.py

"""EPC SAP-schema check for portfolio 805, and whether each is mapper-supported.

For every UPRN currently in the ``property`` table for portfolio 805, look up its
latest EPC certificate's ``schemaType`` (one /api/domestic/search per postcode,
reusing scripts.hyde_epc_schema_versions) and check it against the schemas the
EpcPropertyData mapper actually handles
(``EpcPropertyDataMapper.from_api_response``, datatypes/epc/domain/mapper.py).

Prints a per-schema tally with a supported? flag and an example UPRN, and writes
the full per-UPRN mapping to durkan_805_schema_check.csv.

    python -m scripts.lisasrequest.durkan_805_schema_check
    python -m scripts.lisasrequest.durkan_805_schema_check --portfolio 805 --workers 8

Reads OPEN_EPC_API_TOKEN from backend/.env and POSTGRES_* from the root .env.
Run from the worktree root.
"""

from __future__ import annotations

import argparse
import csv
import os
import sys
from collections import Counter
from pathlib import Path

from dotenv import load_dotenv
from sqlmodel import select

_REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(_REPO_ROOT))  # worktree root first — avoid the import trap

from infrastructure.postgres.config import PostgresConfig  # noqa: E402
from infrastructure.postgres.engine import make_engine, make_session  # noqa: E402
from infrastructure.postgres.property_table import PropertyRow  # noqa: E402
from scripts.fill_domna_addresses import clean_postcode  # noqa: E402
from scripts.hyde_epc_schema_versions import (  # noqa: E402
    NOT_IN_EPC,
    build_uprn_schema_map,
)

# Schemas EpcPropertyDataMapper.from_api_response dispatches on (everything else
# raises "Unsupported EPC schema"). Keep in sync with mapper.py:2539-2603.
SUPPORTED_SCHEMAS = frozenset(
    {
        "RdSAP-Schema-17.0",
        "RdSAP-Schema-17.1",
        "RdSAP-Schema-18.0",
        "RdSAP-Schema-19.0",
        "RdSAP-Schema-20.0.0",
        "RdSAP-Schema-21.0.0",
        "RdSAP-Schema-21.0.1",
        "SAP-Schema-16.0",
        "SAP-Schema-16.2",
        "SAP-Schema-16.3",
        "SAP-Schema-17.0",
        "SAP-Schema-17.1",
        "SAP-Schema-18.0.0",
    }
)

_DEFAULT_OUT = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_805_schema_check.csv"


def load_portfolio_uprns(portfolio_id: int) -> list[tuple[int, str]]:
    """Return (uprn, postcode) for every property in the portfolio with a UPRN."""
    load_dotenv(_REPO_ROOT / ".env")
    engine = make_engine(PostgresConfig.from_env(os.environ))
    session = make_session(engine)
    try:
        stmt = select(PropertyRow.uprn, PropertyRow.postcode).where(
            PropertyRow.portfolio_id == portfolio_id
        )
        out: list[tuple[int, str]] = []
        for uprn, postcode in session.exec(stmt).all():
            if uprn is not None:
                out.append((int(uprn), str(postcode or "")))
        return out
    finally:
        session.close()


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--portfolio", type=int, default=805)
    parser.add_argument("--out", type=Path, default=_DEFAULT_OUT)
    parser.add_argument("--workers", type=int, default=8)
    args = parser.parse_args()

    load_dotenv(_REPO_ROOT / "backend" / ".env")
    token = os.environ.get("OPEN_EPC_API_TOKEN")
    if not token:
        print("OPEN_EPC_API_TOKEN not set (backend/.env)")
        return 2

    pairs = load_portfolio_uprns(args.portfolio)
    postcodes = sorted({clean_postcode(pc) for _, pc in pairs if pc})
    print(
        f"Portfolio {args.portfolio}: {len(pairs)} UPRNs across "
        f"{len(postcodes)} unique postcodes"
    )

    by_uprn = build_uprn_schema_map(postcodes, token, args.workers)
    print(f"EPC search returned a schema for {len(by_uprn)} distinct UPRNs")

    tally: Counter[str] = Counter()
    example: dict[str, int] = {}
    rows_out: list[tuple[int, str, str, str]] = []  # uprn, schema, supported, postcode
    seen: set[int] = set()
    for uprn, pc in pairs:
        if uprn in seen:
            continue
        seen.add(uprn)
        schema = by_uprn.get(uprn, (NOT_IN_EPC, ""))[0]
        supported = "yes" if schema in SUPPORTED_SCHEMAS else "no"
        tally[schema] += 1
        example.setdefault(schema, uprn)
        rows_out.append((uprn, schema, supported, clean_postcode(pc)))

    with args.out.open("w", newline="", encoding="utf-8") as fh:
        writer = csv.writer(fh)
        writer.writerow(["uprn", "schema_version", "mapper_supported", "postcode"])
        writer.writerows(rows_out)

    supported_count = sum(c for s, c in tally.items() if s in SUPPORTED_SCHEMAS)
    print(f"\nSchema versions across {len(seen)} distinct UPRNs in portfolio "
          f"{args.portfolio}:\n")
    print(f"  {'schema version':<26} {'count':>5}  {'supported?':<10} example UPRN")
    print(f"  {'-' * 26} {'-' * 5}  {'-' * 10} {'-' * 12}")
    for schema, count in tally.most_common():
        supported = "yes" if schema in SUPPORTED_SCHEMAS else "NO"
        print(f"  {schema:<26} {count:>5}  {supported:<10} {example[schema]}")
    print(
        f"\nMapper-supported: {supported_count}/{len(seen)} UPRNs. "
        f"Full mapping -> {args.out}"
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())