tweak script to use provided input. track output in md file

scripts/list_properties_by_postcode.py

@@ -0,0 +1,47 @@
+"""Print a dict of postcode → property IDs for a portfolio, sorted by group size.
+
+Edit PORTFOLIO_ID below, then hit Run.
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# CONFIG
+# ---------------------------------------------------------------------------
+PORTFOLIO_ID: int = 796
+# ---------------------------------------------------------------------------
+
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(_REPO_ROOT))
+
+from sqlalchemy import text  # noqa: E402
+
+from scripts.e2e_common import ENV_PATH, build_engine, load_env  # noqa: E402
+
+load_env(ENV_PATH)
+engine = build_engine()
+
+with engine.connect() as conn:
+    rows = conn.execute(
+        text("SELECT id, postcode FROM property WHERE portfolio_id = :pid ORDER BY postcode, id"),
+        {"pid": PORTFOLIO_ID},
+    ).fetchall()
+
+by_postcode: dict[str, list[int]] = defaultdict(list)
+for pid, postcode in rows:
+    by_postcode[postcode or "UNKNOWN"].append(int(pid))
+
+sorted_dict = dict(sorted(by_postcode.items(), key=lambda kv: len(kv[1])))
+
+output_path = _REPO_ROOT / "scripts" / f"properties_by_postcode_{PORTFOLIO_ID}.txt"
+
+lines = [f"{postcode!r}: {ids}" for postcode, ids in sorted_dict.items()]
+lines.append(
+    f"\nTotal postcodes: {len(sorted_dict)}, total properties: {sum(len(v) for v in sorted_dict.values())}"
+)
+output_path.write_text("\n".join(lines), encoding="utf-8")
+print(f"Saved to {output_path}")

scripts/processed_postcodes_796.md

@@ -0,0 +1,8 @@
+# Processed Properties — Portfolio 796
+
+| Postcode | Property IDs | Processed |
+|----------|--------------|-----------|
+| BN10 8EQ | 739489 | 2026-06-23 |
+| BN11 2LT | 740998 | 2026-06-23 |
+| BN11 2NH | 733234 | 2026-06-23 |
+| BN11 4EP | 730259 | 2026-06-23 |

scripts/trigger_modelling_e2e_sqs.py

@@ -1,24 +1,24 @@
-"""Enqueue one SQS message per property for the modelling_e2e Lambda.
+"""Enqueue one SQS message per postcode group for the modelling_e2e Lambda.
 
-Reads all property IDs for the given portfolio from the DB and sends a batch of
-SQS messages, one per property. The Lambda then processes each message
-independently, enabling concurrent modelling at scale.
+Reads postcode → property ID groups from the file produced by
+list_properties_by_postcode.py, skips postcodes already logged in the tracking
+markdown, then sends one SQS message per postcode batch.
 
-Edit the CONFIG block below, then run via VSCode Run button or Jupyter.
-AWS creds come from the ambient ~/.aws profile; DB creds from backend/.env.
+Edit the CONFIG block below, then hit Run.
+AWS creds come from the ambient ~/.aws profile.
 """
 
 from __future__ import annotations
 
-# --------------------------------------------------------------------------
+# ---------------------------------------------------------------------------
 # CONFIG — edit these before running
 # ---------------------------------------------------------------------------
-PORTFOLIO_ID: int = 785
-SCENARIO_ID: int = 1266
-SQS_URL: str = "https://sqs.eu-west-2.amazonaws.com/ACCOUNT_ID/modelling-e2e-STAGE"
+PORTFOLIO_ID: int = 796
+SCENARIO_ID: int = 1268
+SQS_QUEUE_NAME: str = "modelling_e2e-queue-dev"
 
-# Set to a positive integer to enqueue only the first N properties (trial run).
-LIMIT: int | None = 10
+# Number of postcodes to process this run.
+POSTCODES_LIMIT: int = 2
 
 # True → Lambda runs the full pipeline but skips all DB writes (safe for testing).
 DRY_RUN: bool = True
@@ -27,8 +27,10 @@ DRY_RUN: bool = True
 NO_SOLAR: bool = False
 # ---------------------------------------------------------------------------
 
+import ast
 import json
 import sys
+from datetime import date
 from pathlib import Path
 from typing import Any, cast
 from uuid import uuid4
@@ -37,69 +39,97 @@ _REPO_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(_REPO_ROOT))
 
 import boto3  # noqa: E402
-from sqlalchemy import text  # noqa: E402
 
-from scripts.e2e_common import ENV_PATH, build_engine, load_env  # noqa: E402
-
-_BATCH_SIZE = 10
+_POSTCODES_FILE = _REPO_ROOT / "scripts" / f"properties_by_postcode_{PORTFOLIO_ID}.txt"
+_PROCESSED_MD = _REPO_ROOT / "scripts" / f"processed_postcodes_{PORTFOLIO_ID}.md"
 
 
-def _property_ids(portfolio_id: int, limit: int | None, engine: object) -> list[int]:
-    from sqlalchemy.engine import Engine
-
-    assert isinstance(engine, Engine)
-    query = "SELECT id FROM property WHERE portfolio_id = :pid ORDER BY id"
-    if limit is not None:
-        query += f" LIMIT {int(limit)}"
-    with engine.connect() as conn:
-        rows = conn.execute(text(query), {"pid": portfolio_id}).fetchall()
-    return [int(r[0]) for r in rows]
+def _load_postcode_map() -> dict[str, list[int]]:
+    if not _POSTCODES_FILE.exists():
+        raise FileNotFoundError(
+            f"{_POSTCODES_FILE} not found — run list_properties_by_postcode.py first"
+        )
+    result: dict[str, list[int]] = {}
+    for line in _POSTCODES_FILE.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("Total"):
+            continue
+        postcode_repr, ids_repr = line.split(": ", 1)
+        result[ast.literal_eval(postcode_repr)] = ast.literal_eval(ids_repr)
+    return result
 
 
-def _batches(items: list[int], size: int) -> list[list[int]]:
-    return [items[i : i + size] for i in range(0, len(items), size)]
+def _already_processed() -> set[str]:
+    if not _PROCESSED_MD.exists():
+        return set()
+    processed: set[str] = set()
+    for line in _PROCESSED_MD.read_text(encoding="utf-8").splitlines():
+        if (
+            line.startswith("| ")
+            and not line.startswith("| Postcode")
+            and "---" not in line
+        ):
+            parts = [p.strip() for p in line.split("|")]
+            if len(parts) >= 2 and parts[1]:
+                processed.add(parts[1])
+    return processed
+
+
+def _record_processed(postcode: str, ids: list[int]) -> None:
+    today = date.today().isoformat()
+    ids_str = ", ".join(str(i) for i in ids)
+    row = f"| {postcode} | {ids_str} | {today} |\n"
+    if not _PROCESSED_MD.exists():
+        _PROCESSED_MD.write_text(
+            f"# Processed Properties — Portfolio {PORTFOLIO_ID}\n\n"
+            "| Postcode | Property IDs | Processed |\n"
+            "|----------|--------------|-----------|\n" + row,
+            encoding="utf-8",
+        )
+    else:
+        with _PROCESSED_MD.open("a", encoding="utf-8") as f:
+            f.write(row)
 
 
 def main() -> None:
-    load_env(ENV_PATH)
-    engine = build_engine()
+    postcode_map = _load_postcode_map()
+    done = _already_processed()
 
-    ids = _property_ids(PORTFOLIO_ID, LIMIT, engine)
-    if not ids:
-        print(f"no properties found for portfolio {PORTFOLIO_ID}")
+    pending = [(pc, ids) for pc, ids in postcode_map.items() if pc not in done]
+    to_process = pending[:POSTCODES_LIMIT]
+
+    if not to_process:
+        print("Nothing left to process.")
         return
 
+    sqs: Any = cast(
+        Any, boto3.client("sqs", region_name="eu-west-2")
+    )  # pyright: ignore[reportUnknownMemberType]
+    sqs_url: str = sqs.get_queue_url(QueueName=SQS_QUEUE_NAME)["QueueUrl"]
+
     print(
-        f"enqueuing {len(ids)} properties "
+        f"sending {len(to_process)} messages "
         f"(portfolio={PORTFOLIO_ID}, scenario={SCENARIO_ID}, "
-        f"no_solar={NO_SOLAR}, dry_run={DRY_RUN}) → {SQS_URL}"
+        f"dry_run={DRY_RUN}, no_solar={NO_SOLAR}) → {sqs_url}"
     )
 
-    sqs: Any = cast(
-        Any, boto3.client("sqs")
-    )  # pyright: ignore[reportUnknownMemberType]
-    sent = 0
-    for batch in _batches(ids, _BATCH_SIZE):
-        entries = [
-            {
-                "Id": str(uuid4()).replace("-", "")[:8] + str(i),
-                "MessageBody": json.dumps(
-                    {
-                        "property_id": [pid],
-                        "portfolio_id": PORTFOLIO_ID,
-                        "scenario_id": SCENARIO_ID,
-                        "no_solar": NO_SOLAR,
-                        "dry_run": DRY_RUN,
-                    }
-                ),
-            }
-            for i, pid in enumerate(batch)
-        ]
-        sqs.send_message_batch(QueueUrl=SQS_URL, Entries=entries)
-        sent += len(batch)
-        print(f"  sent {sent}/{len(ids)}", end="\r")
+    for postcode, ids in to_process:
+        sqs.send_message(
+            QueueUrl=sqs_url,
+            MessageBody=json.dumps(
+                {
+                    "property_ids": ids,
+                    "portfolio_id": PORTFOLIO_ID,
+                    "scenario_id": SCENARIO_ID,
+                    "no_solar": NO_SOLAR,
+                    "dry_run": DRY_RUN,
+                }
+            ),
+        )
+        _record_processed(postcode, ids)
+        print(f"  sent {postcode}: {ids}")
 
-    print(f"\ndone — {sent} messages enqueued")
+    print(f"\ndone — {len(to_process)} messages enqueued, tracking → {_PROCESSED_MD}")
 
 
 main()