mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
111 lines
3.9 KiB
Python
111 lines
3.9 KiB
Python
"""Step 3 (Durkan portfolio): insert the reshaped rows into the ``property`` table.
|
|
|
|
Reads durkan_finaliser_input.csv (step 2) and, per row, maps it with the real
|
|
finaliser mapper (``BulkUploadFinaliserOrchestrator._row_to_insert``) and inserts
|
|
via the same ``PropertyPostgresRepository.insert_all`` the Lambda uses — so a row
|
|
written here is identical to one the production finaliser would write. Insert is
|
|
ON CONFLICT (portfolio_id, uprn) DO NOTHING, so re-running is safe.
|
|
|
|
DRY RUN BY DEFAULT — it dedupes, reports, and writes the collisions file but does
|
|
NOT touch the database. Add --commit to actually insert.
|
|
|
|
# preview only (no DB writes): dedupe + mapping report
|
|
python -m scripts.lisasrequest.finalise_to_property_table --portfolio 805
|
|
|
|
# actually insert
|
|
python -m scripts.lisasrequest.finalise_to_property_table --portfolio 805 --commit
|
|
|
|
Postgres target comes from the root .env (POSTGRES_*). Run from the worktree root.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(_REPO_ROOT)) # worktree root first — avoid the import trap
|
|
|
|
from scripts.finalise_to_property_table import ( # noqa: E402
|
|
dedupe_by_uprn,
|
|
insert_rows,
|
|
)
|
|
|
|
_DEFAULT_IN = _REPO_ROOT / "scripts" / "lisasrequest" / "durkan_finaliser_input.csv"
|
|
_DEFAULT_COLLISIONS = (
|
|
_REPO_ROOT / "scripts" / "lisasrequest" / "durkan_finaliser_collisions.csv"
|
|
)
|
|
UPRN_COL = "address2uprn_uprn"
|
|
MATCHED_ADDRESS_COL = "address2uprn_address"
|
|
POSTCODE_COL = "postcode"
|
|
LEXISCORE_COL = "address2uprn_lexiscore"
|
|
|
|
|
|
def read_rows(path: Path) -> list[dict[str, str]]:
|
|
with path.open(newline="", encoding="utf-8-sig") as fh:
|
|
return [dict(row) for row in csv.DictReader(fh)]
|
|
|
|
|
|
def _preview(rows: list[dict[str, str]]) -> None:
|
|
"""Show the first few rows as they will be inserted (no DB, no mapper call).
|
|
|
|
The finalise step applies the standard finaliser mapper
|
|
(BulkUploadFinaliserOrchestrator) on insert; the fields below are its inputs.
|
|
"""
|
|
print("\nSample rows to insert (uprn | matched address | postcode | lexiscore):")
|
|
for row in rows[:3]:
|
|
print(
|
|
f" {row.get(UPRN_COL)} | {row.get(MATCHED_ADDRESS_COL)!r} | "
|
|
f"{row.get(POSTCODE_COL)!r} | {row.get(LEXISCORE_COL)}"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--in", dest="inp", type=Path, default=_DEFAULT_IN)
|
|
parser.add_argument("--portfolio", type=int, required=True)
|
|
parser.add_argument(
|
|
"--commit",
|
|
action="store_true",
|
|
help="actually insert into property (default is a dry-run preview)",
|
|
)
|
|
parser.add_argument("--collisions", type=Path, default=_DEFAULT_COLLISIONS)
|
|
args = parser.parse_args()
|
|
|
|
rows = read_rows(args.inp)
|
|
print(f"Loaded {len(rows)} finaliser rows from {args.inp}")
|
|
|
|
kept, dropped = dedupe_by_uprn(rows)
|
|
if dropped:
|
|
with args.collisions.open("w", newline="", encoding="utf-8") as fh:
|
|
writer = csv.DictWriter(fh, fieldnames=list(dropped[0].keys()))
|
|
writer.writeheader()
|
|
writer.writerows(dropped)
|
|
print(
|
|
f"{len(dropped)} duplicate-UPRN rows dropped -> {args.collisions} "
|
|
f"({len(kept)} unique to insert)"
|
|
)
|
|
else:
|
|
print(f"No duplicate-UPRN collisions; {len(kept)} unique rows to insert.")
|
|
|
|
_preview(kept)
|
|
|
|
if not args.commit:
|
|
print(
|
|
f"\nDRY RUN — nothing written. {len(kept)} rows would be inserted into "
|
|
f"portfolio {args.portfolio}. Re-run with --commit to write."
|
|
)
|
|
return 0
|
|
|
|
inserted = insert_rows(kept, args.portfolio)
|
|
print(
|
|
f"\nInserted {inserted} new properties into portfolio {args.portfolio} "
|
|
f"({len(kept) - inserted} already existed; ON CONFLICT DO NOTHING)."
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|