"""EPC SAP-schema check for portfolio 805, and whether each is mapper-supported. For every UPRN currently in the ``property`` table for portfolio 805, look up its latest EPC certificate's ``schemaType`` (one /api/domestic/search per postcode, reusing scripts.hyde_epc_schema_versions) and check it against the schemas the EpcPropertyData mapper actually handles (``EpcPropertyDataMapper.from_api_response``, datatypes/epc/domain/mapper.py). Prints a per-schema tally with a supported? flag and an example UPRN, and writes the full per-UPRN mapping to durkan_805_schema_check.csv. python -m scripts.lisasrequest.durkan_805_schema_check python -m scripts.lisasrequest.durkan_805_schema_check --portfolio 805 --workers 8 Reads OPEN_EPC_API_TOKEN from backend/.env and POSTGRES_* from the root .env. Run from the worktree root. """ from __future__ import annotations import argparse import csv import os import sys from collections import Counter from pathlib import Path from dotenv import load_dotenv from sqlmodel import select _REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(_REPO_ROOT)) # worktree root first — avoid the import trap from infrastructure.postgres.config import PostgresConfig # noqa: E402 from infrastructure.postgres.engine import make_engine, make_session # noqa: E402 from infrastructure.postgres.property_table import PropertyRow # noqa: E402 from scripts.fill_domna_addresses import clean_postcode # noqa: E402 from scripts.hyde_epc_schema_versions import ( # noqa: E402 NOT_IN_EPC, build_uprn_schema_map, ) # Schemas EpcPropertyDataMapper.from_api_response dispatches on (everything else # raises "Unsupported EPC schema"). Keep in sync with mapper.py:2539-2603. SUPPORTED_SCHEMAS = frozenset( { "RdSAP-Schema-17.0", "RdSAP-Schema-17.1", "RdSAP-Schema-18.0", "RdSAP-Schema-19.0", "RdSAP-Schema-20.0.0", "RdSAP-Schema-21.0.0", "RdSAP-Schema-21.0.1", "SAP-Schema-16.0", "SAP-Schema-16.2", "SAP-Schema-16.3", "SAP-Schema-17.0", "SAP-Schema-17.1", "SAP-Schema-18.0.0", } ) _DEFAULT_OUT = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_805_schema_check.csv" def load_portfolio_uprns(portfolio_id: int) -> list[tuple[int, str]]: """Return (uprn, postcode) for every property in the portfolio with a UPRN.""" load_dotenv(_REPO_ROOT / ".env") engine = make_engine(PostgresConfig.from_env(os.environ)) session = make_session(engine) try: stmt = select(PropertyRow.uprn, PropertyRow.postcode).where( PropertyRow.portfolio_id == portfolio_id ) out: list[tuple[int, str]] = [] for uprn, postcode in session.exec(stmt).all(): if uprn is not None: out.append((int(uprn), str(postcode or ""))) return out finally: session.close() def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--portfolio", type=int, default=805) parser.add_argument("--out", type=Path, default=_DEFAULT_OUT) parser.add_argument("--workers", type=int, default=8) args = parser.parse_args() load_dotenv(_REPO_ROOT / "backend" / ".env") token = os.environ.get("OPEN_EPC_API_TOKEN") if not token: print("OPEN_EPC_API_TOKEN not set (backend/.env)") return 2 pairs = load_portfolio_uprns(args.portfolio) postcodes = sorted({clean_postcode(pc) for _, pc in pairs if pc}) print( f"Portfolio {args.portfolio}: {len(pairs)} UPRNs across " f"{len(postcodes)} unique postcodes" ) by_uprn = build_uprn_schema_map(postcodes, token, args.workers) print(f"EPC search returned a schema for {len(by_uprn)} distinct UPRNs") tally: Counter[str] = Counter() example: dict[str, int] = {} rows_out: list[tuple[int, str, str, str]] = [] # uprn, schema, supported, postcode seen: set[int] = set() for uprn, pc in pairs: if uprn in seen: continue seen.add(uprn) schema = by_uprn.get(uprn, (NOT_IN_EPC, ""))[0] supported = "yes" if schema in SUPPORTED_SCHEMAS else "no" tally[schema] += 1 example.setdefault(schema, uprn) rows_out.append((uprn, schema, supported, clean_postcode(pc))) with args.out.open("w", newline="", encoding="utf-8") as fh: writer = csv.writer(fh) writer.writerow(["uprn", "schema_version", "mapper_supported", "postcode"]) writer.writerows(rows_out) supported_count = sum(c for s, c in tally.items() if s in SUPPORTED_SCHEMAS) print(f"\nSchema versions across {len(seen)} distinct UPRNs in portfolio " f"{args.portfolio}:\n") print(f" {'schema version':<26} {'count':>5} {'supported?':<10} example UPRN") print(f" {'-' * 26} {'-' * 5} {'-' * 10} {'-' * 12}") for schema, count in tally.most_common(): supported = "yes" if schema in SUPPORTED_SCHEMAS else "NO" print(f" {schema:<26} {count:>5} {supported:<10} {example[schema]}") print( f"\nMapper-supported: {supported_count}/{len(seen)} UPRNs. " f"Full mapping -> {args.out}" ) return 0 if __name__ == "__main__": sys.exit(main())