script

scripts/backfill_site_notes_epc_property_uprn.py

@@ -0,0 +1,188 @@
+"""Backfill ``epc_property.uprn`` on PasHub site-notes rows written before the
+UPRN-persistence fix (#1538). Targets have ``uprn IS NULL`` and a non-null
+``uploaded_file_id`` (the site-notes signature; these rows carry no ``property_id``
+/ ``portfolio_id``).
+
+For each target the UPRN is recovered, in priority order, from (1)
+``uploaded_files.uprn``, then when that's null (2) ``property.landlord_property_id``
+or (3) ``hubspot_deal_data.uprn`` (via ``uploaded_files.hubspot_deal_id``). Every
+method is validated against the target portfolio's ``property`` table — the only
+handle for scoping these portfolio-less rows — and the written value is that
+property row's ``uprn``. Rows matching no property, or ambiguously matching several
+UPRNs, are skipped and reported. ``uprn_source`` is left untouched, as in #1538.
+
+Set ``PORTFOLIO_ID`` below before running. DRY-RUN by default (writes nothing);
+pass ``--apply`` to execute in a transaction. Idempotent — re-running is a no-op.
+
+Run with:  ``python -m scripts.backfill_site_notes_epc_property_uprn [--apply]``
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from sqlalchemy import Connection, text
+
+from scripts.e2e_common import build_engine, load_env
+
+# ---------------------------------------------------------------------------
+# Set this to the portfolio you are backfilling before running the script.
+# ---------------------------------------------------------------------------
+PORTFOLIO_ID: int = 0  # TODO: set the target portfolio id
+
+
+# Target rows: site-notes-mapped epc_property rows still missing a UPRN. The
+# non-null uploaded_file_id is the site-notes signature (predicted / lodged-EPC
+# rows don't have one), and uprn IS NULL is the pre-#1538 gap.
+#
+# For each we join uploaded_files (the only link to a property), the deal row
+# (hubspot_deal_id -> hubspot_deal_data.deal_id), and the target portfolio's
+# property table, resolving the UPRN from uploaded_files.uprn or, when that's null,
+# via landlord_property_id or the deal's uprn. The value written is always the
+# property row's uprn. GROUP BY ... HAVING count(DISTINCT p.uprn) = 1 drops any row
+# that resolves to more than one portfolio UPRN (ambiguous — left for a human).
+# hubspot_deal_data.uprn is text, so it's numeric-guarded (^[0-9]+$) before casting.
+_RESOLVED_CTE = """
+    WITH target AS (
+        SELECT e.id AS epc_property_id, e.uploaded_file_id
+          FROM epc_property e
+         WHERE e.uploaded_file_id IS NOT NULL
+           AND e.uprn IS NULL
+    ),
+    resolved AS (
+        SELECT t.epc_property_id,
+               min(p.uprn) AS resolved_uprn,
+               bool_or(uf.uprn IS NOT NULL AND p.uprn = uf.uprn) AS via_uploaded_file,
+               bool_or(uf.uprn IS NULL
+                       AND uf.landlord_property_id IS NOT NULL
+                       AND p.landlord_property_id = uf.landlord_property_id
+                       ) AS via_property_lpi,
+               bool_or(uf.uprn IS NULL
+                       AND btrim(hdd.uprn) ~ '^[0-9]+$'
+                       AND p.uprn = btrim(hdd.uprn)::bigint
+                       ) AS via_hubspot_deal,
+               count(DISTINCT p.uprn) AS distinct_uprns
+          FROM target t
+          JOIN uploaded_files uf ON uf.id = t.uploaded_file_id
+          LEFT JOIN hubspot_deal_data hdd ON hdd.deal_id = uf.hubspot_deal_id
+          JOIN property p
+            ON p.portfolio_id = :portfolio_id
+           AND p.uprn IS NOT NULL
+           AND (
+                 (uf.uprn IS NOT NULL AND p.uprn = uf.uprn)
+              OR (uf.uprn IS NULL
+                  AND uf.landlord_property_id IS NOT NULL
+                  AND p.landlord_property_id = uf.landlord_property_id)
+              OR (uf.uprn IS NULL
+                  AND btrim(hdd.uprn) ~ '^[0-9]+$'
+                  AND p.uprn = btrim(hdd.uprn)::bigint)
+               )
+         GROUP BY t.epc_property_id
+    )
+"""
+
+_REPORT = text(
+    _RESOLVED_CTE
+    + """
+    SELECT
+        (SELECT count(*) FROM target)                                       AS targets,
+        count(*) FILTER (WHERE distinct_uprns = 1)                          AS resolvable,
+        count(*) FILTER (WHERE distinct_uprns = 1
+                         AND via_uploaded_file)                             AS via_uploaded_file,
+        count(*) FILTER (WHERE distinct_uprns = 1
+                         AND NOT via_uploaded_file
+                         AND via_property_lpi)                              AS via_property_lpi,
+        count(*) FILTER (WHERE distinct_uprns = 1
+                         AND NOT via_uploaded_file
+                         AND NOT via_property_lpi
+                         AND via_hubspot_deal)                              AS via_hubspot_deal,
+        count(*) FILTER (WHERE distinct_uprns > 1)                          AS ambiguous
+      FROM resolved
+    """
+)
+
+_UPDATE = text(
+    _RESOLVED_CTE
+    + """
+    UPDATE epc_property e
+       SET uprn = r.resolved_uprn
+      FROM resolved r
+     WHERE e.id = r.epc_property_id
+       AND r.distinct_uprns = 1
+       AND e.uprn IS NULL
+    """
+)
+
+
+def backfill(conn: Connection, *, portfolio_id: int, apply: bool) -> dict[str, int]:
+    """Report (and, when ``apply``, execute) the UPRN backfill for ``portfolio_id``.
+
+    Returns the diagnostic counts: total ``targets`` (site-notes rows missing a
+    UPRN), how many are ``resolvable`` within the portfolio (split by the method
+    that resolved them — ``via_uploaded_file`` / ``via_property_lpi`` /
+    ``via_hubspot_deal``, in that priority order), how many are ``ambiguous``
+    (skipped), and — when applied — how many rows were ``updated``."""
+    row = conn.execute(_REPORT, {"portfolio_id": portfolio_id}).one()
+    counts = {
+        "targets": row.targets,
+        "resolvable": row.resolvable,
+        "via_uploaded_file": row.via_uploaded_file,
+        "via_property_lpi": row.via_property_lpi,
+        "via_hubspot_deal": row.via_hubspot_deal,
+        "ambiguous": row.ambiguous,
+    }
+    if apply:
+        result = conn.execute(_UPDATE, {"portfolio_id": portfolio_id})
+        counts["updated"] = result.rowcount
+    return counts
+
+
+def main() -> None:
+    load_env()
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="execute the updates (default: dry-run, writes nothing)",
+    )
+    args = parser.parse_args()
+
+    if PORTFOLIO_ID <= 0:
+        raise SystemExit(
+            "Set PORTFOLIO_ID at the top of this script to the target portfolio "
+            "before running."
+        )
+
+    engine = build_engine()
+    with engine.begin() as conn:
+        conn.execute(text("SET statement_timeout = 120000"))
+        counts = backfill(conn, portfolio_id=PORTFOLIO_ID, apply=args.apply)
+
+    unresolved = counts["targets"] - counts["resolvable"] - counts["ambiguous"]
+    verb = "backfilled" if args.apply else "would backfill"
+    print(f"Portfolio {PORTFOLIO_ID}: {counts['targets']} site-notes epc_property "
+          f"row(s) missing a UPRN.")
+    print(
+        f"  {verb} {counts['resolvable']} row(s) "
+        f"({counts['via_uploaded_file']} via uploaded_files.uprn, "
+        f"{counts['via_property_lpi']} via property.landlord_property_id, "
+        f"{counts['via_hubspot_deal']} via hubspot_deal_data.uprn)."
+    )
+    if counts["ambiguous"]:
+        print(
+            f"  SKIPPED {counts['ambiguous']} row(s) that matched more than one "
+            "portfolio UPRN (ambiguous — resolve by hand)."
+        )
+    if unresolved:
+        print(
+            f"  SKIPPED {unresolved} row(s) with no matching property in this "
+            "portfolio (no uploaded_files.uprn, landlord_property_id or deal match)."
+        )
+    if args.apply:
+        print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.")
+    else:
+        print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
+
+
+if __name__ == "__main__":
+    main()