"""Step 3 (Durkan portfolio): insert the reshaped rows into the ``property`` table. Reads durkan_finaliser_input.csv (step 2) and, per row, maps it with the real finaliser mapper (``BulkUploadFinaliserOrchestrator._row_to_insert``) and inserts via the same ``PropertyPostgresRepository.insert_all`` the Lambda uses — so a row written here is identical to one the production finaliser would write. Insert is ON CONFLICT (portfolio_id, uprn) DO NOTHING, so re-running is safe. DRY RUN BY DEFAULT — it dedupes, reports, and writes the collisions file but does NOT touch the database. Add --commit to actually insert. # preview only (no DB writes): dedupe + mapping report python -m scripts.lisasrequest.finalise_to_property_table --portfolio 805 # actually insert python -m scripts.lisasrequest.finalise_to_property_table --portfolio 805 --commit Postgres target comes from the root .env (POSTGRES_*). Run from the worktree root. """ from __future__ import annotations import argparse import csv import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(_REPO_ROOT)) # worktree root first — avoid the import trap from scripts.finalise_to_property_table import ( # noqa: E402 dedupe_by_uprn, insert_rows, ) _DEFAULT_IN = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_finaliser_input.csv" _DEFAULT_COLLISIONS = ( _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_finaliser_collisions.csv" ) UPRN_COL = "address2uprn_uprn" MATCHED_ADDRESS_COL = "address2uprn_address" POSTCODE_COL = "postcode" LEXISCORE_COL = "address2uprn_lexiscore" def read_rows(path: Path) -> list[dict[str, str]]: with path.open(newline="", encoding="utf-8-sig") as fh: return [dict(row) for row in csv.DictReader(fh)] def _preview(rows: list[dict[str, str]]) -> None: """Show the first few rows as they will be inserted (no DB, no mapper call). The finalise step applies the standard finaliser mapper (BulkUploadFinaliserOrchestrator) on insert; the fields below are its inputs. """ print("\nSample rows to insert (uprn | matched address | postcode | lexiscore):") for row in rows[:3]: print( f" {row.get(UPRN_COL)} | {row.get(MATCHED_ADDRESS_COL)!r} | " f"{row.get(POSTCODE_COL)!r} | {row.get(LEXISCORE_COL)}" ) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--in", dest="inp", type=Path, default=_DEFAULT_IN) parser.add_argument("--portfolio", type=int, required=True) parser.add_argument( "--commit", action="store_true", help="actually insert into property (default is a dry-run preview)", ) parser.add_argument("--collisions", type=Path, default=_DEFAULT_COLLISIONS) args = parser.parse_args() rows = read_rows(args.inp) print(f"Loaded {len(rows)} finaliser rows from {args.inp}") kept, dropped = dedupe_by_uprn(rows) if dropped: with args.collisions.open("w", newline="", encoding="utf-8") as fh: writer = csv.DictWriter(fh, fieldnames=list(dropped[0].keys())) writer.writeheader() writer.writerows(dropped) print( f"{len(dropped)} duplicate-UPRN rows dropped -> {args.collisions} " f"({len(kept)} unique to insert)" ) else: print(f"No duplicate-UPRN collisions; {len(kept)} unique rows to insert.") _preview(kept) if not args.commit: print( f"\nDRY RUN — nothing written. {len(kept)} rows would be inserted into " f"portfolio {args.portfolio}. Re-run with --commit to write." ) return 0 inserted = insert_rows(kept, args.portfolio) print( f"\nInserted {inserted} new properties into portfolio {args.portfolio} " f"({len(kept) - inserted} already existed; ON CONFLICT DO NOTHING)." ) return 0 if __name__ == "__main__": sys.exit(main())