mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-27 23:35:01 +00:00
update script
This commit is contained in:
Daniel Roth
parent
08e11538cd
commit
f5d2bdd651
1 changed files with 118 additions and 107 deletions
|
|
@ -3,185 +3,196 @@ UPRN-persistence fix (#1538). Targets have ``uprn IS NULL`` and a non-null
|
|||
``uploaded_file_id`` (the site-notes signature; these rows carry no ``property_id``
|
||||
/ ``portfolio_id``).
|
||||
|
||||
For each target the UPRN is recovered, in priority order, from (1)
|
||||
``uploaded_files.uprn``, then when that's null (2) ``property.landlord_property_id``
|
||||
or (3) ``hubspot_deal_data.uprn`` (via ``uploaded_files.hubspot_deal_id``). Every
|
||||
method is validated against the target portfolio's ``property`` table — the only
|
||||
handle for scoping these portfolio-less rows — and the written value is that
|
||||
property row's ``uprn``. Rows matching no property, or ambiguously matching several
|
||||
UPRNs, are skipped and reported. ``uprn_source`` is left untouched, as in #1538.
|
||||
The run is scoped to the rows whose deal (``uploaded_files.hubspot_deal_id`` →
|
||||
``hubspot_deal_data.deal_id``) carries ``PROJECT_CODE``. For each such target the
|
||||
UPRN is taken, in priority order, from (1) ``uploaded_files.uprn``, then when that's
|
||||
null (2) ``hubspot_deal_data.uprn`` (text, so numeric-guarded and cast). A row is
|
||||
skipped and reported when neither yields a UPRN, or when the deal offers several
|
||||
different UPRNs (ambiguous). ``uprn_source`` is left untouched, as in #1538.
|
||||
|
||||
Set ``PORTFOLIO_ID`` below before running. DRY-RUN by default (writes nothing);
|
||||
pass ``--apply`` to execute in a transaction. Idempotent — re-running is a no-op.
|
||||
|
||||
Run with: ``python -m scripts.backfill_site_notes_epc_property_uprn [--apply]``
|
||||
Set ``PROJECT_CODE`` and ``DRY_RUN`` below, then run the file (VS Code ▷ Run, or
|
||||
``python -m scripts.backfill_site_notes_epc_property_uprn``). With ``DRY_RUN = True``
|
||||
it prints what it would change and writes nothing; set it to ``False`` to execute in
|
||||
a transaction. Idempotent — re-running is a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import Connection, text
|
||||
|
||||
from scripts.e2e_common import build_engine, load_env
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Set this to the portfolio you are backfilling before running the script.
|
||||
# Set these before running.
|
||||
# ---------------------------------------------------------------------------
|
||||
PORTFOLIO_ID: int = 0 # TODO: set the target portfolio id
|
||||
PROJECT_CODE: str = (
|
||||
"[The Guinness Partnership GMCA Bid - WH:SF - 052026 - 1219680020683]" # TODO: hubspot_deal_data.project_code to scope to
|
||||
)
|
||||
DRY_RUN: bool = True # True = report only (writes nothing); False = apply the update
|
||||
|
||||
|
||||
# Target rows: site-notes-mapped epc_property rows still missing a UPRN. The
|
||||
# non-null uploaded_file_id is the site-notes signature (predicted / lodged-EPC
|
||||
# rows don't have one), and uprn IS NULL is the pre-#1538 gap.
|
||||
# Target rows: site-notes-mapped epc_property rows still missing a UPRN, scoped to
|
||||
# the deals carrying PROJECT_CODE. The non-null uploaded_file_id is the site-notes
|
||||
# signature (predicted / lodged-EPC rows don't have one; the JOIN to uploaded_files
|
||||
# enforces it), and uprn IS NULL is the pre-#1538 gap. The EXISTS restricts to rows
|
||||
# whose deal (uploaded_files.hubspot_deal_id -> hubspot_deal_data.deal_id) has the
|
||||
# requested project_code.
|
||||
#
|
||||
# For each we join uploaded_files (the only link to a property), the deal row
|
||||
# (hubspot_deal_id -> hubspot_deal_data.deal_id), and the target portfolio's
|
||||
# property table, resolving the UPRN from uploaded_files.uprn or, when that's null,
|
||||
# via landlord_property_id or the deal's uprn. The value written is always the
|
||||
# property row's uprn. GROUP BY ... HAVING count(DISTINCT p.uprn) = 1 drops any row
|
||||
# that resolves to more than one portfolio UPRN (ambiguous — left for a human).
|
||||
# hubspot_deal_data.uprn is text, so it's numeric-guarded (^[0-9]+$) before casting.
|
||||
# resolved: per target, the UPRN candidates — uploaded_files.uprn (single, via the
|
||||
# PK join) and the deal's uprn. hubspot_deal_data.uprn is text, so it's
|
||||
# numeric-guarded (^[0-9]+$) before casting; a deal_id can have several rows, so we
|
||||
# take the min and also count the distinct values to spot ambiguity.
|
||||
#
|
||||
# final: the value written is uploaded_files.uprn if present, else the deal's uprn.
|
||||
# via_uploaded_file / via_hubspot_deal record which source won. A row is ambiguous
|
||||
# (skipped) when it falls back to the deal and the deal offers >1 distinct UPRN.
|
||||
_RESOLVED_CTE = """
|
||||
WITH target AS (
|
||||
SELECT e.id AS epc_property_id, e.uploaded_file_id
|
||||
FROM epc_property e
|
||||
WHERE e.uploaded_file_id IS NOT NULL
|
||||
AND e.uprn IS NULL
|
||||
JOIN uploaded_files uf ON uf.id = e.uploaded_file_id
|
||||
WHERE e.uprn IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM hubspot_deal_data d
|
||||
WHERE d.deal_id = uf.hubspot_deal_id
|
||||
AND d.project_code = :project_code
|
||||
)
|
||||
),
|
||||
resolved AS (
|
||||
SELECT t.epc_property_id,
|
||||
min(p.uprn) AS resolved_uprn,
|
||||
bool_or(uf.uprn IS NOT NULL AND p.uprn = uf.uprn) AS via_uploaded_file,
|
||||
bool_or(uf.uprn IS NULL
|
||||
AND uf.landlord_property_id IS NOT NULL
|
||||
AND p.landlord_property_id = uf.landlord_property_id
|
||||
) AS via_property_lpi,
|
||||
bool_or(uf.uprn IS NULL
|
||||
AND btrim(hdd.uprn) ~ '^[0-9]+$'
|
||||
AND p.uprn = btrim(hdd.uprn)::bigint
|
||||
) AS via_hubspot_deal,
|
||||
count(DISTINCT p.uprn) AS distinct_uprns
|
||||
uf.uprn AS uploaded_uprn,
|
||||
min(CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$'
|
||||
THEN btrim(hdd.uprn)::bigint END) AS deal_uprn,
|
||||
count(DISTINCT CASE WHEN btrim(hdd.uprn) ~ '^[0-9]+$'
|
||||
THEN btrim(hdd.uprn)::bigint END) AS distinct_deal_uprns
|
||||
FROM target t
|
||||
JOIN uploaded_files uf ON uf.id = t.uploaded_file_id
|
||||
LEFT JOIN hubspot_deal_data hdd ON hdd.deal_id = uf.hubspot_deal_id
|
||||
JOIN property p
|
||||
ON p.portfolio_id = :portfolio_id
|
||||
AND p.uprn IS NOT NULL
|
||||
AND (
|
||||
(uf.uprn IS NOT NULL AND p.uprn = uf.uprn)
|
||||
OR (uf.uprn IS NULL
|
||||
AND uf.landlord_property_id IS NOT NULL
|
||||
AND p.landlord_property_id = uf.landlord_property_id)
|
||||
OR (uf.uprn IS NULL
|
||||
AND btrim(hdd.uprn) ~ '^[0-9]+$'
|
||||
AND p.uprn = btrim(hdd.uprn)::bigint)
|
||||
)
|
||||
GROUP BY t.epc_property_id
|
||||
GROUP BY t.epc_property_id, uf.uprn
|
||||
),
|
||||
final AS (
|
||||
SELECT epc_property_id,
|
||||
COALESCE(uploaded_uprn, deal_uprn) AS resolved_uprn,
|
||||
uploaded_uprn IS NOT NULL AS via_uploaded_file,
|
||||
(uploaded_uprn IS NULL AND deal_uprn IS NOT NULL) AS via_hubspot_deal,
|
||||
(uploaded_uprn IS NULL AND distinct_deal_uprns > 1) AS ambiguous
|
||||
FROM resolved
|
||||
)
|
||||
"""
|
||||
|
||||
_REPORT = text(
|
||||
_RESOLVED_CTE
|
||||
+ """
|
||||
_REPORT = text(_RESOLVED_CTE + """
|
||||
SELECT
|
||||
(SELECT count(*) FROM target) AS targets,
|
||||
count(*) FILTER (WHERE distinct_uprns = 1) AS resolvable,
|
||||
count(*) FILTER (WHERE distinct_uprns = 1
|
||||
count(*) FILTER (WHERE resolved_uprn IS NOT NULL
|
||||
AND NOT ambiguous) AS resolvable,
|
||||
count(*) FILTER (WHERE resolved_uprn IS NOT NULL
|
||||
AND NOT ambiguous
|
||||
AND via_uploaded_file) AS via_uploaded_file,
|
||||
count(*) FILTER (WHERE distinct_uprns = 1
|
||||
AND NOT via_uploaded_file
|
||||
AND via_property_lpi) AS via_property_lpi,
|
||||
count(*) FILTER (WHERE distinct_uprns = 1
|
||||
AND NOT via_uploaded_file
|
||||
AND NOT via_property_lpi
|
||||
count(*) FILTER (WHERE resolved_uprn IS NOT NULL
|
||||
AND NOT ambiguous
|
||||
AND via_hubspot_deal) AS via_hubspot_deal,
|
||||
count(*) FILTER (WHERE distinct_uprns > 1) AS ambiguous
|
||||
FROM resolved
|
||||
"""
|
||||
)
|
||||
count(*) FILTER (WHERE ambiguous) AS ambiguous
|
||||
FROM final
|
||||
""")
|
||||
|
||||
_UPDATE = text(
|
||||
_RESOLVED_CTE
|
||||
+ """
|
||||
_UPDATE = text(_RESOLVED_CTE + """
|
||||
UPDATE epc_property e
|
||||
SET uprn = r.resolved_uprn
|
||||
FROM resolved r
|
||||
WHERE e.id = r.epc_property_id
|
||||
AND r.distinct_uprns = 1
|
||||
SET uprn = f.resolved_uprn
|
||||
FROM final f
|
||||
WHERE e.id = f.epc_property_id
|
||||
AND f.resolved_uprn IS NOT NULL
|
||||
AND NOT f.ambiguous
|
||||
AND e.uprn IS NULL
|
||||
"""
|
||||
)
|
||||
""")
|
||||
|
||||
# Addresses of the rows we can't backfill (no UPRN from either source), so they can
|
||||
# be chased up by hand before / after a live run.
|
||||
_UNRESOLVED = text(_RESOLVED_CTE + """
|
||||
SELECT e.id AS epc_property_id,
|
||||
e.address_line_1, e.address_line_2, e.post_town, e.postcode
|
||||
FROM final f
|
||||
JOIN epc_property e ON e.id = f.epc_property_id
|
||||
WHERE f.resolved_uprn IS NULL
|
||||
ORDER BY e.postcode, e.address_line_1
|
||||
""")
|
||||
|
||||
|
||||
def backfill(conn: Connection, *, portfolio_id: int, apply: bool) -> dict[str, int]:
|
||||
"""Report (and, when ``apply``, execute) the UPRN backfill for ``portfolio_id``.
|
||||
def _unresolved_addresses(conn: Connection, *, project_code: str) -> list[str]:
|
||||
"""One formatted ``"epc_property <id>: <address>"`` line per target row that
|
||||
yielded no UPRN from either source."""
|
||||
lines: list[str] = []
|
||||
for r in conn.execute(_UNRESOLVED, {"project_code": project_code}):
|
||||
parts = [r.address_line_1, r.address_line_2, r.post_town, r.postcode]
|
||||
address = ", ".join(p for p in parts if p) or "(no address on epc_property)"
|
||||
lines.append(f"epc_property {r.epc_property_id}: {address}")
|
||||
return lines
|
||||
|
||||
|
||||
def backfill(conn: Connection, *, project_code: str, apply: bool) -> dict[str, int]:
|
||||
"""Report (and, when ``apply``, execute) the UPRN backfill for the deals
|
||||
carrying ``project_code``.
|
||||
|
||||
Returns the diagnostic counts: total ``targets`` (site-notes rows missing a
|
||||
UPRN), how many are ``resolvable`` within the portfolio (split by the method
|
||||
that resolved them — ``via_uploaded_file`` / ``via_property_lpi`` /
|
||||
``via_hubspot_deal``, in that priority order), how many are ``ambiguous``
|
||||
(skipped), and — when applied — how many rows were ``updated``."""
|
||||
row = conn.execute(_REPORT, {"portfolio_id": portfolio_id}).one()
|
||||
UPRN), how many are ``resolvable`` (split by the source that resolved them —
|
||||
``via_uploaded_file`` then ``via_hubspot_deal``, in that priority order), how
|
||||
many are ``ambiguous`` (skipped), and — when applied — how many rows were
|
||||
``updated``."""
|
||||
params = {"project_code": project_code}
|
||||
row = conn.execute(_REPORT, params).one()
|
||||
counts = {
|
||||
"targets": row.targets,
|
||||
"resolvable": row.resolvable,
|
||||
"via_uploaded_file": row.via_uploaded_file,
|
||||
"via_property_lpi": row.via_property_lpi,
|
||||
"via_hubspot_deal": row.via_hubspot_deal,
|
||||
"ambiguous": row.ambiguous,
|
||||
}
|
||||
if apply:
|
||||
result = conn.execute(_UPDATE, {"portfolio_id": portfolio_id})
|
||||
result = conn.execute(_UPDATE, params)
|
||||
counts["updated"] = result.rowcount
|
||||
return counts
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_env()
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="execute the updates (default: dry-run, writes nothing)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if PORTFOLIO_ID <= 0:
|
||||
if not PROJECT_CODE:
|
||||
raise SystemExit(
|
||||
"Set PORTFOLIO_ID at the top of this script to the target portfolio "
|
||||
"before running."
|
||||
"Set PROJECT_CODE at the top of this script to the hubspot_deal_data "
|
||||
"project_code to scope to before running."
|
||||
)
|
||||
|
||||
engine = build_engine()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("SET statement_timeout = 120000"))
|
||||
counts = backfill(conn, portfolio_id=PORTFOLIO_ID, apply=args.apply)
|
||||
counts = backfill(conn, project_code=PROJECT_CODE, apply=not DRY_RUN)
|
||||
unresolved_addresses = _unresolved_addresses(conn, project_code=PROJECT_CODE)
|
||||
|
||||
unresolved = counts["targets"] - counts["resolvable"] - counts["ambiguous"]
|
||||
verb = "backfilled" if args.apply else "would backfill"
|
||||
print(f"Portfolio {PORTFOLIO_ID}: {counts['targets']} site-notes epc_property "
|
||||
f"row(s) missing a UPRN.")
|
||||
verb = "would backfill" if DRY_RUN else "backfilled"
|
||||
print(
|
||||
f"Project {PROJECT_CODE!r}: {counts['targets']} site-notes epc_property "
|
||||
f"row(s) missing a UPRN."
|
||||
)
|
||||
print(
|
||||
f" {verb} {counts['resolvable']} row(s) "
|
||||
f"({counts['via_uploaded_file']} via uploaded_files.uprn, "
|
||||
f"{counts['via_property_lpi']} via property.landlord_property_id, "
|
||||
f"{counts['via_hubspot_deal']} via hubspot_deal_data.uprn)."
|
||||
)
|
||||
if counts["ambiguous"]:
|
||||
print(
|
||||
f" SKIPPED {counts['ambiguous']} row(s) that matched more than one "
|
||||
"portfolio UPRN (ambiguous — resolve by hand)."
|
||||
f" SKIPPED {counts['ambiguous']} row(s) whose deal offers more than one "
|
||||
"distinct UPRN (ambiguous — resolve by hand)."
|
||||
)
|
||||
if unresolved:
|
||||
print(
|
||||
f" SKIPPED {unresolved} row(s) with no matching property in this "
|
||||
"portfolio (no uploaded_files.uprn, landlord_property_id or deal match)."
|
||||
f" SKIPPED {unresolved} row(s) with no UPRN available "
|
||||
"(no uploaded_files.uprn and no numeric deal uprn):"
|
||||
)
|
||||
if args.apply:
|
||||
print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.")
|
||||
for line in unresolved_addresses:
|
||||
print(f" {line}")
|
||||
if DRY_RUN:
|
||||
print("\nDRY-RUN — nothing written. Set DRY_RUN = False to execute.")
|
||||
else:
|
||||
print("\nDRY-RUN — nothing written. Re-run with --apply to execute.")
|
||||
print(f"\nApplied. {counts.get('updated', 0)} row(s) updated.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue