From 08e11538cd0ff3fb02a99d83cdae5ebb75cb6682 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 09:13:12 +0000 Subject: [PATCH 1/2] script --- .../backfill_site_notes_epc_property_uprn.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 scripts/backfill_site_notes_epc_property_uprn.py diff --git a/scripts/backfill_site_notes_epc_property_uprn.py b/scripts/backfill_site_notes_epc_property_uprn.py new file mode 100644 index 000000000..8e097e5fc --- /dev/null +++ b/scripts/backfill_site_notes_epc_property_uprn.py @@ -0,0 +1,188 @@ +"""Backfill ``epc_property.uprn`` on PasHub site-notes rows written before the +UPRN-persistence fix (#1538). Targets have ``uprn IS NULL`` and a non-null +``uploaded_file_id`` (the site-notes signature; these rows carry no ``property_id`` +/ ``portfolio_id``). + +For each target the UPRN is recovered, in priority order, from (1) +``uploaded_files.uprn``, then when that's null (2) ``property.landlord_property_id`` +or (3) ``hubspot_deal_data.uprn`` (via ``uploaded_files.hubspot_deal_id``). Every +method is validated against the target portfolio's ``property`` table — the only +handle for scoping these portfolio-less rows — and the written value is that +property row's ``uprn``. Rows matching no property, or ambiguously matching several +UPRNs, are skipped and reported. ``uprn_source`` is left untouched, as in #1538. + +Set ``PORTFOLIO_ID`` below before running. DRY-RUN by default (writes nothing); +pass ``--apply`` to execute in a transaction. Idempotent — re-running is a no-op. + +Run with: ``python -m scripts.backfill_site_notes_epc_property_uprn [--apply]`` +""" + +from __future__ import annotations + +import argparse + +from sqlalchemy import Connection, text + +from scripts.e2e_common import build_engine, load_env + +# --------------------------------------------------------------------------- +# Set this to the portfolio you are backfilling before running the script. +# --------------------------------------------------------------------------- +PORTFOLIO_ID: int = 0 # TODO: set the target portfolio id + + +# Target rows: site-notes-mapped epc_property rows still missing a UPRN. The +# non-null uploaded_file_id is the site-notes signature (predicted / lodged-EPC +# rows don't have one), and uprn IS NULL is the pre-#1538 gap. +# +# For each we join uploaded_files (the only link to a property), the deal row +# (hubspot_deal_id -> hubspot_deal_data.deal_id), and the target portfolio's +# property table, resolving the UPRN from uploaded_files.uprn or, when that's null, +# via landlord_property_id or the deal's uprn. The value written is always the +# property row's uprn. GROUP BY ... HAVING count(DISTINCT p.uprn) = 1 drops any row +# that resolves to more than one portfolio UPRN (ambiguous — left for a human). +# hubspot_deal_data.uprn is text, so it's numeric-guarded (^[0-9]+$) before casting. +_RESOLVED_CTE = """ + WITH target AS ( + SELECT e.id AS epc_property_id, e.uploaded_file_id + FROM epc_property e + WHERE e.uploaded_file_id IS NOT NULL + AND e.uprn IS NULL + ), + resolved AS ( + SELECT t.epc_property_id, + min(p.uprn) AS resolved_uprn, + bool_or(uf.uprn IS NOT NULL AND p.uprn = uf.uprn) AS via_uploaded_file, + bool_or(uf.uprn IS NULL + AND uf.landlord_property_id IS NOT NULL + AND p.landlord_property_id = uf.landlord_property_id + ) AS via_property_lpi, + bool_or(uf.uprn IS NULL + AND btrim(hdd.uprn) ~ '^[0-9]+$' + AND p.uprn = btrim(hdd.uprn)::bigint + ) AS via_hubspot_deal, + count(DISTINCT p.uprn) AS distinct_uprns + FROM target t + JOIN uploaded_files uf ON uf.id = t.uploaded_file_id + LEFT JOIN hubspot_deal_data hdd ON hdd.deal_id = uf.hubspot_deal_id + JOIN property p + ON p.portfolio_id = :portfolio_id + AND p.uprn IS NOT NULL + AND ( + (uf.uprn IS NOT NULL AND p.uprn = uf.uprn) + OR (uf.uprn IS NULL + AND uf.landlord_property_id IS NOT NULL + AND p.landlord_property_id = uf.landlord_property_id) + OR (uf.uprn IS NULL + AND btrim(hdd.uprn) ~ '^[0-9]+$' + AND p.uprn = btrim(hdd.uprn)::bigint) + ) + GROUP BY t.epc_property_id + ) +""" + +_REPORT = text( + _RESOLVED_CTE + + """ + SELECT + (SELECT count(*) FROM target) AS targets, + count(*) FILTER (WHERE distinct_uprns = 1) AS resolvable, + count(*) FILTER (WHERE distinct_uprns = 1 + AND via_uploaded_file) AS via_uploaded_file, + count(*) FILTER (WHERE distinct_uprns = 1 + AND NOT via_uploaded_file + AND via_property_lpi) AS via_property_lpi, + count(*) FILTER (WHERE distinct_uprns = 1 + AND NOT via_uploaded_file + AND NOT via_property_lpi + AND via_hubspot_deal) AS via_hubspot_deal, + count(*) FILTER (WHERE distinct_uprns > 1) AS ambiguous + FROM resolved + """ +) + +_UPDATE = text( + _RESOLVED_CTE + + """ + UPDATE epc_property e + SET uprn = r.resolved_uprn + FROM resolved r + WHERE e.id = r.epc_property_id + AND r.distinct_uprns = 1 + AND e.uprn IS NULL + """ +) + + +def backfill(conn: Connection, *, portfolio_id: int, apply: bool) -> dict[str, int]: + """Report (and, when ``apply``, execute) the UPRN backfill for ``portfolio_id``. + + Returns the diagnostic counts: total ``targets`` (site-notes rows missing a + UPRN), how many are ``resolvable`` within the portfolio (split by the method + that resolved them — ``via_uploaded_file`` / ``via_property_lpi`` / + ``via_hubspot_deal``, in that priority order), how many are ``ambiguous`` + (skipped), and — when applied — how many rows were ``updated``.""" + row = conn.execute(_REPORT, {"portfolio_id": portfolio_id}).one() + counts = { + "targets": row.targets, + "resolvable": row.resolvable, + "via_uploaded_file": row.via_uploaded_file, + "via_property_lpi": row.via_property_lpi, + "via_hubspot_deal": row.via_hubspot_deal, + "ambiguous": row.ambiguous, + } + if apply: + result = conn.execute(_UPDATE, {"portfolio_id": portfolio_id}) + counts["updated"] = result.rowcount + return counts + + +def main() -> None: + load_env() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="execute the updates (default: dry-run, writes nothing)", + ) + args = parser.parse_args() + + if PORTFOLIO_ID <= 0: + raise SystemExit( + "Set PORTFOLIO_ID at the top of this script to the target portfolio " + "before running." + ) + + engine = build_engine() + with engine.begin() as conn: + conn.execute(text("SET statement_timeout = 120000")) + counts = backfill(conn, portfolio_id=PORTFOLIO_ID, apply=args.apply) + + unresolved = counts["targets"] - counts["resolvable"] - counts["ambiguous"] + verb = "backfilled" if args.apply else "would backfill" + print(f"Portfolio {PORTFOLIO_ID}: {counts['targets']} site-notes epc_property " + f"row(s) missing a UPRN.") + print( + f" {verb} {counts['resolvable']} row(s) " + f"({counts['via_uploaded_file']} via uploaded_files.uprn, " + f"{counts['via_property_lpi']} via property.landlord_property_id, " + f"{counts['via_hubspot_deal']} via hubspot_deal_data.uprn)." + ) + if counts["ambiguous"]: + print( + f" SKIPPED {counts['ambiguous']} row(s) that matched more than one " + "portfolio UPRN (ambiguous — resolve by hand)." + ) + if unresolved: + print( + f" SKIPPED {unresolved} row(s) with no matching property in this " + "portfolio (no uploaded_files.uprn, landlord_property_id or deal match)." + ) + if args.apply: + print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.") + else: + print("\nDRY-RUN — nothing written. Re-run with --apply to execute.") + + +if __name__ == "__main__": + main() From f5d2bdd65162018fe84df9bbe21d5d341efbb4f6 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 09:40:38 +0000 Subject: [PATCH 2/2] update script --- .../backfill_site_notes_epc_property_uprn.py | 225 +++++++++--------- 1 file changed, 118 insertions(+), 107 deletions(-) diff --git a/scripts/backfill_site_notes_epc_property_uprn.py b/scripts/backfill_site_notes_epc_property_uprn.py index 8e097e5fc..e99d4a5be 100644 --- a/scripts/backfill_site_notes_epc_property_uprn.py +++ b/scripts/backfill_site_notes_epc_property_uprn.py @@ -3,185 +3,196 @@ UPRN-persistence fix (#1538). Targets have ``uprn IS NULL`` and a non-null ``uploaded_file_id`` (the site-notes signature; these rows carry no ``property_id`` / ``portfolio_id``). -For each target the UPRN is recovered, in priority order, from (1) -``uploaded_files.uprn``, then when that's null (2) ``property.landlord_property_id`` -or (3) ``hubspot_deal_data.uprn`` (via ``uploaded_files.hubspot_deal_id``). Every -method is validated against the target portfolio's ``property`` table — the only -handle for scoping these portfolio-less rows — and the written value is that -property row's ``uprn``. Rows matching no property, or ambiguously matching several -UPRNs, are skipped and reported. ``uprn_source`` is left untouched, as in #1538. +The run is scoped to the rows whose deal (``uploaded_files.hubspot_deal_id`` → +``hubspot_deal_data.deal_id``) carries ``PROJECT_CODE``. For each such target the +UPRN is taken, in priority order, from (1) ``uploaded_files.uprn``, then when that's +null (2) ``hubspot_deal_data.uprn`` (text, so numeric-guarded and cast). A row is +skipped and reported when neither yields a UPRN, or when the deal offers several +different UPRNs (ambiguous). ``uprn_source`` is left untouched, as in #1538. -Set ``PORTFOLIO_ID`` below before running. DRY-RUN by default (writes nothing); -pass ``--apply`` to execute in a transaction. Idempotent — re-running is a no-op. - -Run with: ``python -m scripts.backfill_site_notes_epc_property_uprn [--apply]`` +Set ``PROJECT_CODE`` and ``DRY_RUN`` below, then run the file (VS Code ▷ Run, or +``python -m scripts.backfill_site_notes_epc_property_uprn``). With ``DRY_RUN = True`` +it prints what it would change and writes nothing; set it to ``False`` to execute in +a transaction. Idempotent — re-running is a no-op. """ from __future__ import annotations -import argparse - from sqlalchemy import Connection, text from scripts.e2e_common import build_engine, load_env # --------------------------------------------------------------------------- -# Set this to the portfolio you are backfilling before running the script. +# Set these before running. # --------------------------------------------------------------------------- -PORTFOLIO_ID: int = 0 # TODO: set the target portfolio id +PROJECT_CODE: str = ( + "[The Guinness Partnership GMCA Bid - WH:SF - 052026 - 1219680020683]" # TODO: hubspot_deal_data.project_code to scope to +) +DRY_RUN: bool = True # True = report only (writes nothing); False = apply the update -# Target rows: site-notes-mapped epc_property rows still missing a UPRN. The -# non-null uploaded_file_id is the site-notes signature (predicted / lodged-EPC -# rows don't have one), and uprn IS NULL is the pre-#1538 gap. +# Target rows: site-notes-mapped epc_property rows still missing a UPRN, scoped to +# the deals carrying PROJECT_CODE. The non-null uploaded_file_id is the site-notes +# signature (predicted / lodged-EPC rows don't have one; the JOIN to uploaded_files +# enforces it), and uprn IS NULL is the pre-#1538 gap. The EXISTS restricts to rows +# whose deal (uploaded_files.hubspot_deal_id -> hubspot_deal_data.deal_id) has the +# requested project_code. # -# For each we join uploaded_files (the only link to a property), the deal row -# (hubspot_deal_id -> hubspot_deal_data.deal_id), and the target portfolio's -# property table, resolving the UPRN from uploaded_files.uprn or, when that's null, -# via landlord_property_id or the deal's uprn. The value written is always the -# property row's uprn. GROUP BY ... HAVING count(DISTINCT p.uprn) = 1 drops any row -# that resolves to more than one portfolio UPRN (ambiguous — left for a human). -# hubspot_deal_data.uprn is text, so it's numeric-guarded (^[0-9]+$) before casting. +# resolved: per target, the UPRN candidates — uploaded_files.uprn (single, via the +# PK join) and the deal's uprn. hubspot_deal_data.uprn is text, so it's +# numeric-guarded (^[0-9]+$) before casting; a deal_id can have several rows, so we +# take the min and also count the distinct values to spot ambiguity. +# +# final: the value written is uploaded_files.uprn if present, else the deal's uprn. +# via_uploaded_file / via_hubspot_deal record which source won. A row is ambiguous +# (skipped) when it falls back to the deal and the deal offers >1 distinct UPRN. _RESOLVED_CTE = """ WITH target AS ( SELECT e.id AS epc_property_id, e.uploaded_file_id FROM epc_property e - WHERE e.uploaded_file_id IS NOT NULL - AND e.uprn IS NULL + JOIN uploaded_files uf ON uf.id = e.uploaded_file_id + WHERE e.uprn IS NULL + AND EXISTS ( + SELECT 1 FROM hubspot_deal_data d + WHERE d.deal_id = uf.hubspot_deal_id + AND d.project_code = :project_code + ) ), resolved AS ( SELECT t.epc_property_id, - min(p.uprn) AS resolved_uprn, - bool_or(uf.uprn IS NOT NULL AND p.uprn = uf.uprn) AS via_uploaded_file, - bool_or(uf.uprn IS NULL - AND uf.landlord_property_id IS NOT NULL - AND p.landlord_property_id = uf.landlord_property_id - ) AS via_property_lpi, - bool_or(uf.uprn IS NULL - AND btrim(hdd.uprn) ~ '^[0-9]+$' - AND p.uprn = btrim(hdd.uprn)::bigint - ) AS via_hubspot_deal, - count(DISTINCT p.uprn) AS distinct_uprns + uf.uprn AS uploaded_uprn, + min(CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$' + THEN btrim(hdd.uprn)::bigint END) AS deal_uprn, + count(DISTINCT CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$' + THEN btrim(hdd.uprn)::bigint END) AS distinct_deal_uprns FROM target t JOIN uploaded_files uf ON uf.id = t.uploaded_file_id LEFT JOIN hubspot_deal_data hdd ON hdd.deal_id = uf.hubspot_deal_id - JOIN property p - ON p.portfolio_id = :portfolio_id - AND p.uprn IS NOT NULL - AND ( - (uf.uprn IS NOT NULL AND p.uprn = uf.uprn) - OR (uf.uprn IS NULL - AND uf.landlord_property_id IS NOT NULL - AND p.landlord_property_id = uf.landlord_property_id) - OR (uf.uprn IS NULL - AND btrim(hdd.uprn) ~ '^[0-9]+$' - AND p.uprn = btrim(hdd.uprn)::bigint) - ) - GROUP BY t.epc_property_id + GROUP BY t.epc_property_id, uf.uprn + ), + final AS ( + SELECT epc_property_id, + COALESCE(uploaded_uprn, deal_uprn) AS resolved_uprn, + uploaded_uprn IS NOT NULL AS via_uploaded_file, + (uploaded_uprn IS NULL AND deal_uprn IS NOT NULL) AS via_hubspot_deal, + (uploaded_uprn IS NULL AND distinct_deal_uprns > 1) AS ambiguous + FROM resolved ) """ -_REPORT = text( - _RESOLVED_CTE - + """ +_REPORT = text(_RESOLVED_CTE + """ SELECT (SELECT count(*) FROM target) AS targets, - count(*) FILTER (WHERE distinct_uprns = 1) AS resolvable, - count(*) FILTER (WHERE distinct_uprns = 1 + count(*) FILTER (WHERE resolved_uprn IS NOT NULL + AND NOT ambiguous) AS resolvable, + count(*) FILTER (WHERE resolved_uprn IS NOT NULL + AND NOT ambiguous AND via_uploaded_file) AS via_uploaded_file, - count(*) FILTER (WHERE distinct_uprns = 1 - AND NOT via_uploaded_file - AND via_property_lpi) AS via_property_lpi, - count(*) FILTER (WHERE distinct_uprns = 1 - AND NOT via_uploaded_file - AND NOT via_property_lpi + count(*) FILTER (WHERE resolved_uprn IS NOT NULL + AND NOT ambiguous AND via_hubspot_deal) AS via_hubspot_deal, - count(*) FILTER (WHERE distinct_uprns > 1) AS ambiguous - FROM resolved - """ -) + count(*) FILTER (WHERE ambiguous) AS ambiguous + FROM final + """) -_UPDATE = text( - _RESOLVED_CTE - + """ +_UPDATE = text(_RESOLVED_CTE + """ UPDATE epc_property e - SET uprn = r.resolved_uprn - FROM resolved r - WHERE e.id = r.epc_property_id - AND r.distinct_uprns = 1 + SET uprn = f.resolved_uprn + FROM final f + WHERE e.id = f.epc_property_id + AND f.resolved_uprn IS NOT NULL + AND NOT f.ambiguous AND e.uprn IS NULL - """ -) + """) + +# Addresses of the rows we can't backfill (no UPRN from either source), so they can +# be chased up by hand before / after a live run. +_UNRESOLVED = text(_RESOLVED_CTE + """ + SELECT e.id AS epc_property_id, + e.address_line_1, e.address_line_2, e.post_town, e.postcode + FROM final f + JOIN epc_property e ON e.id = f.epc_property_id + WHERE f.resolved_uprn IS NULL + ORDER BY e.postcode, e.address_line_1 + """) -def backfill(conn: Connection, *, portfolio_id: int, apply: bool) -> dict[str, int]: - """Report (and, when ``apply``, execute) the UPRN backfill for ``portfolio_id``. +def _unresolved_addresses(conn: Connection, *, project_code: str) -> list[str]: + """One formatted ``"epc_property :
"`` line per target row that + yielded no UPRN from either source.""" + lines: list[str] = [] + for r in conn.execute(_UNRESOLVED, {"project_code": project_code}): + parts = [r.address_line_1, r.address_line_2, r.post_town, r.postcode] + address = ", ".join(p for p in parts if p) or "(no address on epc_property)" + lines.append(f"epc_property {r.epc_property_id}: {address}") + return lines + + +def backfill(conn: Connection, *, project_code: str, apply: bool) -> dict[str, int]: + """Report (and, when ``apply``, execute) the UPRN backfill for the deals + carrying ``project_code``. Returns the diagnostic counts: total ``targets`` (site-notes rows missing a - UPRN), how many are ``resolvable`` within the portfolio (split by the method - that resolved them — ``via_uploaded_file`` / ``via_property_lpi`` / - ``via_hubspot_deal``, in that priority order), how many are ``ambiguous`` - (skipped), and — when applied — how many rows were ``updated``.""" - row = conn.execute(_REPORT, {"portfolio_id": portfolio_id}).one() + UPRN), how many are ``resolvable`` (split by the source that resolved them — + ``via_uploaded_file`` then ``via_hubspot_deal``, in that priority order), how + many are ``ambiguous`` (skipped), and — when applied — how many rows were + ``updated``.""" + params = {"project_code": project_code} + row = conn.execute(_REPORT, params).one() counts = { "targets": row.targets, "resolvable": row.resolvable, "via_uploaded_file": row.via_uploaded_file, - "via_property_lpi": row.via_property_lpi, "via_hubspot_deal": row.via_hubspot_deal, "ambiguous": row.ambiguous, } if apply: - result = conn.execute(_UPDATE, {"portfolio_id": portfolio_id}) + result = conn.execute(_UPDATE, params) counts["updated"] = result.rowcount return counts def main() -> None: load_env() - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--apply", - action="store_true", - help="execute the updates (default: dry-run, writes nothing)", - ) - args = parser.parse_args() - - if PORTFOLIO_ID <= 0: + if not PROJECT_CODE: raise SystemExit( - "Set PORTFOLIO_ID at the top of this script to the target portfolio " - "before running." + "Set PROJECT_CODE at the top of this script to the hubspot_deal_data " + "project_code to scope to before running." ) engine = build_engine() with engine.begin() as conn: conn.execute(text("SET statement_timeout = 120000")) - counts = backfill(conn, portfolio_id=PORTFOLIO_ID, apply=args.apply) + counts = backfill(conn, project_code=PROJECT_CODE, apply=not DRY_RUN) + unresolved_addresses = _unresolved_addresses(conn, project_code=PROJECT_CODE) unresolved = counts["targets"] - counts["resolvable"] - counts["ambiguous"] - verb = "backfilled" if args.apply else "would backfill" - print(f"Portfolio {PORTFOLIO_ID}: {counts['targets']} site-notes epc_property " - f"row(s) missing a UPRN.") + verb = "would backfill" if DRY_RUN else "backfilled" + print( + f"Project {PROJECT_CODE!r}: {counts['targets']} site-notes epc_property " + f"row(s) missing a UPRN." + ) print( f" {verb} {counts['resolvable']} row(s) " f"({counts['via_uploaded_file']} via uploaded_files.uprn, " - f"{counts['via_property_lpi']} via property.landlord_property_id, " f"{counts['via_hubspot_deal']} via hubspot_deal_data.uprn)." ) if counts["ambiguous"]: print( - f" SKIPPED {counts['ambiguous']} row(s) that matched more than one " - "portfolio UPRN (ambiguous — resolve by hand)." + f" SKIPPED {counts['ambiguous']} row(s) whose deal offers more than one " + "distinct UPRN (ambiguous — resolve by hand)." ) if unresolved: print( - f" SKIPPED {unresolved} row(s) with no matching property in this " - "portfolio (no uploaded_files.uprn, landlord_property_id or deal match)." + f" SKIPPED {unresolved} row(s) with no UPRN available " + "(no uploaded_files.uprn and no numeric deal uprn):" ) - if args.apply: - print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.") + for line in unresolved_addresses: + print(f" {line}") + if DRY_RUN: + print("\nDRY-RUN — nothing written. Set DRY_RUN = False to execute.") else: - print("\nDRY-RUN — nothing written. Re-run with --apply to execute.") + print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.") if __name__ == "__main__":