"""Enqueue one SQS message per postcode group for the modelling_e2e Lambda. Reads postcode → property ID groups from the file produced by list_properties_by_postcode.py, skips postcodes already logged in the tracking markdown, then sends one SQS message per postcode batch. Edit the CONFIG block below, then hit Run. AWS creds come from the ambient ~/.aws profile. """ from __future__ import annotations from utilities.logger import setup_logger # --------------------------------------------------------------------------- # CONFIG — edit these before running # --------------------------------------------------------------------------- PORTFOLIO_ID: int = 796 SCENARIO_ID: int = 1268 SQS_QUEUE_NAME: str = "modelling_e2e-queue-dev" # Number of postcodes to process this run. POSTCODES_LIMIT: int = 50 # True → Lambda runs the full pipeline but skips all DB writes (safe for testing). DRY_RUN: bool = False # True → Lambda skips the Google Solar fetch. NO_SOLAR: bool = False # --------------------------------------------------------------------------- import ast import json import sys from datetime import date from pathlib import Path from typing import Any, cast from uuid import uuid4 _REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(_REPO_ROOT)) import boto3 # noqa: E402 logger = setup_logger() _POSTCODES_FILE = _REPO_ROOT / "scripts" / f"properties_by_postcode_{PORTFOLIO_ID}.txt" _PROCESSED_MD = _REPO_ROOT / "scripts" / f"processed_postcodes_{PORTFOLIO_ID}.md" def _load_postcode_map() -> dict[str, list[int]]: if not _POSTCODES_FILE.exists(): raise FileNotFoundError( f"{_POSTCODES_FILE} not found — run list_properties_by_postcode.py first" ) result: dict[str, list[int]] = {} for line in _POSTCODES_FILE.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line or line.startswith("Total"): continue postcode_repr, ids_repr = line.split(": ", 1) result[ast.literal_eval(postcode_repr)] = ast.literal_eval(ids_repr) return result def _already_processed() -> set[str]: if not _PROCESSED_MD.exists(): return set() processed: set[str] = set() for line in _PROCESSED_MD.read_text(encoding="utf-8").splitlines(): if ( line.startswith("| ") and not line.startswith("| Postcode") and "---" not in line ): parts = [p.strip() for p in line.split("|")] if len(parts) >= 2 and parts[1]: processed.add(parts[1]) return processed def _record_processed(postcode: str, ids: list[int]) -> None: today = date.today().isoformat() ids_str = ", ".join(str(i) for i in ids) row = f"| {postcode} | {ids_str} | {today} |\n" if not _PROCESSED_MD.exists(): _PROCESSED_MD.write_text( f"# Processed Properties — Portfolio {PORTFOLIO_ID}\n\n" "| Postcode | Property IDs | Processed |\n" "|----------|--------------|-----------|\n" + row, encoding="utf-8", ) else: with _PROCESSED_MD.open("a", encoding="utf-8") as f: f.write(row) def main() -> None: postcode_map = _load_postcode_map() done = _already_processed() pending = [(pc, ids) for pc, ids in postcode_map.items() if pc not in done] to_process = pending[:POSTCODES_LIMIT] if not to_process: logger.info("Nothing left to process.") return sqs: Any = cast( Any, boto3.client("sqs", region_name="eu-west-2") ) # pyright: ignore[reportUnknownMemberType] sqs_url: str = sqs.get_queue_url(QueueName=SQS_QUEUE_NAME)["QueueUrl"] logger.info( f"sending {len(to_process)} messages " f"(portfolio={PORTFOLIO_ID}, scenario={SCENARIO_ID}, " f"dry_run={DRY_RUN}, no_solar={NO_SOLAR}) → {sqs_url}" ) for postcode, ids in to_process: sqs.send_message( QueueUrl=sqs_url, MessageBody=json.dumps( { "property_ids": ids, "portfolio_id": PORTFOLIO_ID, "scenario_id": SCENARIO_ID, "no_solar": NO_SOLAR, "dry_run": DRY_RUN, } ), ) _record_processed(postcode, ids) logger.info(f" sent {postcode}: {ids}") logger.info( f"\ndone — {len(to_process)} messages enqueued, tracking → {_PROCESSED_MD}" ) main()