update script

scripts/backfill_site_notes_epc_property_uprn.py

@@ -3,185 +3,196 @@ UPRN-persistence fix (#1538). Targets have ``uprn IS NULL`` and a non-null
 ``uploaded_file_id`` (the site-notes signature; these rows carry no ``property_id``
 / ``portfolio_id``).
 
-For each target the UPRN is recovered, in priority order, from (1)
+The run is scoped to the rows whose deal (``uploaded_files.hubspot_deal_id`` →
-``uploaded_files.uprn``, then when that's null (2) ``property.landlord_property_id``
+``hubspot_deal_data.deal_id``) carries ``PROJECT_CODE``. For each such target the
-or (3) ``hubspot_deal_data.uprn`` (via ``uploaded_files.hubspot_deal_id``). Every
+UPRN is taken, in priority order, from (1) ``uploaded_files.uprn``, then when that's
-method is validated against the target portfolio's ``property`` table — the only
+null (2) ``hubspot_deal_data.uprn`` (text, so numeric-guarded and cast). A row is
-handle for scoping these portfolio-less rows — and the written value is that
+skipped and reported when neither yields a UPRN, or when the deal offers several
-property row's ``uprn``. Rows matching no property, or ambiguously matching several
+different UPRNs (ambiguous). ``uprn_source`` is left untouched, as in #1538.
-UPRNs, are skipped and reported. ``uprn_source`` is left untouched, as in #1538.
 
-Set ``PORTFOLIO_ID`` below before running. DRY-RUN by default (writes nothing);
+Set ``PROJECT_CODE`` and ``DRY_RUN`` below, then run the file (VS Code ▷ Run, or
-pass ``--apply`` to execute in a transaction. Idempotent — re-running is a no-op.
+``python -m scripts.backfill_site_notes_epc_property_uprn``). With ``DRY_RUN = True``
-
+it prints what it would change and writes nothing; set it to ``False`` to execute in
-Run with:  ``python -m scripts.backfill_site_notes_epc_property_uprn [--apply]``
+a transaction. Idempotent — re-running is a no-op.
 """
 
 from __future__ import annotations
 
-import argparse
-
 from sqlalchemy import Connection, text
 
 from scripts.e2e_common import build_engine, load_env
 
 # ---------------------------------------------------------------------------
-# Set this to the portfolio you are backfilling before running the script.
+# Set these before running.
 # ---------------------------------------------------------------------------
-PORTFOLIO_ID: int = 0  # TODO: set the target portfolio id
+PROJECT_CODE: str = (
+    "[The Guinness Partnership GMCA Bid - WH:SF - 052026 - 1219680020683]"  # TODO: hubspot_deal_data.project_code to scope to
+)
+DRY_RUN: bool = True  # True = report only (writes nothing); False = apply the update
 
 
-# Target rows: site-notes-mapped epc_property rows still missing a UPRN. The
+# Target rows: site-notes-mapped epc_property rows still missing a UPRN, scoped to
-# non-null uploaded_file_id is the site-notes signature (predicted / lodged-EPC
+# the deals carrying PROJECT_CODE. The non-null uploaded_file_id is the site-notes
-# rows don't have one), and uprn IS NULL is the pre-#1538 gap.
+# signature (predicted / lodged-EPC rows don't have one; the JOIN to uploaded_files
+# enforces it), and uprn IS NULL is the pre-#1538 gap. The EXISTS restricts to rows
+# whose deal (uploaded_files.hubspot_deal_id -> hubspot_deal_data.deal_id) has the
+# requested project_code.
 #
-# For each we join uploaded_files (the only link to a property), the deal row
+# resolved: per target, the UPRN candidates — uploaded_files.uprn (single, via the
-# (hubspot_deal_id -> hubspot_deal_data.deal_id), and the target portfolio's
+# PK join) and the deal's uprn. hubspot_deal_data.uprn is text, so it's
-# property table, resolving the UPRN from uploaded_files.uprn or, when that's null,
+# numeric-guarded (^[0-9]+$) before casting; a deal_id can have several rows, so we
-# via landlord_property_id or the deal's uprn. The value written is always the
+# take the min and also count the distinct values to spot ambiguity.
-# property row's uprn. GROUP BY ... HAVING count(DISTINCT p.uprn) = 1 drops any row
+#
-# that resolves to more than one portfolio UPRN (ambiguous — left for a human).
+# final: the value written is uploaded_files.uprn if present, else the deal's uprn.
-# hubspot_deal_data.uprn is text, so it's numeric-guarded (^[0-9]+$) before casting.
+# via_uploaded_file / via_hubspot_deal record which source won. A row is ambiguous
+# (skipped) when it falls back to the deal and the deal offers >1 distinct UPRN.
 _RESOLVED_CTE = """
     WITH target AS (
         SELECT e.id AS epc_property_id, e.uploaded_file_id
           FROM epc_property e
-         WHERE e.uploaded_file_id IS NOT NULL
+          JOIN uploaded_files uf ON uf.id = e.uploaded_file_id
-           AND e.uprn IS NULL
+         WHERE e.uprn IS NULL
+           AND EXISTS (
+                 SELECT 1 FROM hubspot_deal_data d
+                  WHERE d.deal_id = uf.hubspot_deal_id
+                    AND d.project_code = :project_code
+               )
     ),
     resolved AS (
         SELECT t.epc_property_id,
-               min(p.uprn) AS resolved_uprn,
+               uf.uprn AS uploaded_uprn,
-               bool_or(uf.uprn IS NOT NULL AND p.uprn = uf.uprn) AS via_uploaded_file,
+               min(CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$'
-               bool_or(uf.uprn IS NULL
+                        THEN btrim(hdd.uprn)::bigint END) AS deal_uprn,
-                       AND uf.landlord_property_id IS NOT NULL
+               count(DISTINCT CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$'
-                       AND p.landlord_property_id = uf.landlord_property_id
+                                   THEN btrim(hdd.uprn)::bigint END) AS distinct_deal_uprns
-                       ) AS via_property_lpi,
-               bool_or(uf.uprn IS NULL
-                       AND btrim(hdd.uprn) ~ '^[0-9]+$'
-                       AND p.uprn = btrim(hdd.uprn)::bigint
-                       ) AS via_hubspot_deal,
-               count(DISTINCT p.uprn) AS distinct_uprns
           FROM target t
           JOIN uploaded_files uf ON uf.id = t.uploaded_file_id
           LEFT JOIN hubspot_deal_data hdd ON hdd.deal_id = uf.hubspot_deal_id
-          JOIN property p
+         GROUP BY t.epc_property_id, uf.uprn
-            ON p.portfolio_id = :portfolio_id
+    ),
-           AND p.uprn IS NOT NULL
+    final AS (
-           AND (
+        SELECT epc_property_id,
-                 (uf.uprn IS NOT NULL AND p.uprn = uf.uprn)
+               COALESCE(uploaded_uprn, deal_uprn) AS resolved_uprn,
-              OR (uf.uprn IS NULL
+               uploaded_uprn IS NOT NULL AS via_uploaded_file,
-                  AND uf.landlord_property_id IS NOT NULL
+               (uploaded_uprn IS NULL AND deal_uprn IS NOT NULL) AS via_hubspot_deal,
-                  AND p.landlord_property_id = uf.landlord_property_id)
+               (uploaded_uprn IS NULL AND distinct_deal_uprns > 1) AS ambiguous
-              OR (uf.uprn IS NULL
+          FROM resolved
-                  AND btrim(hdd.uprn) ~ '^[0-9]+$'
-                  AND p.uprn = btrim(hdd.uprn)::bigint)
-               )
-         GROUP BY t.epc_property_id
     )
 """
 
-_REPORT = text(
+_REPORT = text(_RESOLVED_CTE + """
-    _RESOLVED_CTE
-    + """
     SELECT
         (SELECT count(*) FROM target)                                       AS targets,
-        count(*) FILTER (WHERE distinct_uprns = 1)                          AS resolvable,
+        count(*) FILTER (WHERE resolved_uprn IS NOT NULL
-        count(*) FILTER (WHERE distinct_uprns = 1
+                         AND NOT ambiguous)                                 AS resolvable,
+        count(*) FILTER (WHERE resolved_uprn IS NOT NULL
+                         AND NOT ambiguous
                          AND via_uploaded_file)                             AS via_uploaded_file,
-        count(*) FILTER (WHERE distinct_uprns = 1
+        count(*) FILTER (WHERE resolved_uprn IS NOT NULL
-                         AND NOT via_uploaded_file
+                         AND NOT ambiguous
-                         AND via_property_lpi)                              AS via_property_lpi,
-        count(*) FILTER (WHERE distinct_uprns = 1
-                         AND NOT via_uploaded_file
-                         AND NOT via_property_lpi
                          AND via_hubspot_deal)                              AS via_hubspot_deal,
-        count(*) FILTER (WHERE distinct_uprns > 1)                          AS ambiguous
+        count(*) FILTER (WHERE ambiguous)                                   AS ambiguous
-      FROM resolved
+      FROM final
-    """
+    """)
-)
 
-_UPDATE = text(
+_UPDATE = text(_RESOLVED_CTE + """
-    _RESOLVED_CTE
-    + """
     UPDATE epc_property e
-       SET uprn = r.resolved_uprn
+       SET uprn = f.resolved_uprn
-      FROM resolved r
+      FROM final f
-     WHERE e.id = r.epc_property_id
+     WHERE e.id = f.epc_property_id
-       AND r.distinct_uprns = 1
+       AND f.resolved_uprn IS NOT NULL
+       AND NOT f.ambiguous
        AND e.uprn IS NULL
-    """
+    """)
-)
+
+# Addresses of the rows we can't backfill (no UPRN from either source), so they can
+# be chased up by hand before / after a live run.
+_UNRESOLVED = text(_RESOLVED_CTE + """
+    SELECT e.id AS epc_property_id,
+           e.address_line_1, e.address_line_2, e.post_town, e.postcode
+      FROM final f
+      JOIN epc_property e ON e.id = f.epc_property_id
+     WHERE f.resolved_uprn IS NULL
+     ORDER BY e.postcode, e.address_line_1
+    """)
 
 
-def backfill(conn: Connection, *, portfolio_id: int, apply: bool) -> dict[str, int]:
+def _unresolved_addresses(conn: Connection, *, project_code: str) -> list[str]:
-    """Report (and, when ``apply``, execute) the UPRN backfill for ``portfolio_id``.
+    """One formatted ``"epc_property <id>: <address>"`` line per target row that
+    yielded no UPRN from either source."""
+    lines: list[str] = []
+    for r in conn.execute(_UNRESOLVED, {"project_code": project_code}):
+        parts = [r.address_line_1, r.address_line_2, r.post_town, r.postcode]
+        address = ", ".join(p for p in parts if p) or "(no address on epc_property)"
+        lines.append(f"epc_property {r.epc_property_id}: {address}")
+    return lines
+
+
+def backfill(conn: Connection, *, project_code: str, apply: bool) -> dict[str, int]:
+    """Report (and, when ``apply``, execute) the UPRN backfill for the deals
+    carrying ``project_code``.
 
     Returns the diagnostic counts: total ``targets`` (site-notes rows missing a
-    UPRN), how many are ``resolvable`` within the portfolio (split by the method
+    UPRN), how many are ``resolvable`` (split by the source that resolved them —
-    that resolved them — ``via_uploaded_file`` / ``via_property_lpi`` /
+    ``via_uploaded_file`` then ``via_hubspot_deal``, in that priority order), how
-    ``via_hubspot_deal``, in that priority order), how many are ``ambiguous``
+    many are ``ambiguous`` (skipped), and — when applied — how many rows were
-    (skipped), and — when applied — how many rows were ``updated``."""
+    ``updated``."""
-    row = conn.execute(_REPORT, {"portfolio_id": portfolio_id}).one()
+    params = {"project_code": project_code}
+    row = conn.execute(_REPORT, params).one()
     counts = {
         "targets": row.targets,
         "resolvable": row.resolvable,
         "via_uploaded_file": row.via_uploaded_file,
-        "via_property_lpi": row.via_property_lpi,
         "via_hubspot_deal": row.via_hubspot_deal,
         "ambiguous": row.ambiguous,
     }
     if apply:
-        result = conn.execute(_UPDATE, {"portfolio_id": portfolio_id})
+        result = conn.execute(_UPDATE, params)
         counts["updated"] = result.rowcount
     return counts
 
 
 def main() -> None:
     load_env()
-    parser = argparse.ArgumentParser(description=__doc__)
+    if not PROJECT_CODE:
-    parser.add_argument(
-        "--apply",
-        action="store_true",
-        help="execute the updates (default: dry-run, writes nothing)",
-    )
-    args = parser.parse_args()
-
-    if PORTFOLIO_ID <= 0:
         raise SystemExit(
-            "Set PORTFOLIO_ID at the top of this script to the target portfolio "
+            "Set PROJECT_CODE at the top of this script to the hubspot_deal_data "
-            "before running."
+            "project_code to scope to before running."
         )
 
     engine = build_engine()
     with engine.begin() as conn:
         conn.execute(text("SET statement_timeout = 120000"))
-        counts = backfill(conn, portfolio_id=PORTFOLIO_ID, apply=args.apply)
+        counts = backfill(conn, project_code=PROJECT_CODE, apply=not DRY_RUN)
+        unresolved_addresses = _unresolved_addresses(conn, project_code=PROJECT_CODE)
 
     unresolved = counts["targets"] - counts["resolvable"] - counts["ambiguous"]
-    verb = "backfilled" if args.apply else "would backfill"
+    verb = "would backfill" if DRY_RUN else "backfilled"
-    print(f"Portfolio {PORTFOLIO_ID}: {counts['targets']} site-notes epc_property "
+    print(
-          f"row(s) missing a UPRN.")
+        f"Project {PROJECT_CODE!r}: {counts['targets']} site-notes epc_property "
+        f"row(s) missing a UPRN."
+    )
     print(
         f"  {verb} {counts['resolvable']} row(s) "
         f"({counts['via_uploaded_file']} via uploaded_files.uprn, "
-        f"{counts['via_property_lpi']} via property.landlord_property_id, "
         f"{counts['via_hubspot_deal']} via hubspot_deal_data.uprn)."
     )
     if counts["ambiguous"]:
         print(
-            f"  SKIPPED {counts['ambiguous']} row(s) that matched more than one "
+            f"  SKIPPED {counts['ambiguous']} row(s) whose deal offers more than one "
-            "portfolio UPRN (ambiguous — resolve by hand)."
+            "distinct UPRN (ambiguous — resolve by hand)."
         )
     if unresolved:
         print(
-            f"  SKIPPED {unresolved} row(s) with no matching property in this "
+            f"  SKIPPED {unresolved} row(s) with no UPRN available "
-            "portfolio (no uploaded_files.uprn, landlord_property_id or deal match)."
+            "(no uploaded_files.uprn and no numeric deal uprn):"
         )
-    if args.apply:
+        for line in unresolved_addresses:
-        print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.")
+            print(f"      {line}")
+    if DRY_RUN:
+        print("\nDRY-RUN — nothing written. Set DRY_RUN = False to execute.")
     else:
-        print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
+        print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.")
 
 
 if __name__ == "__main__":