Model/scripts/trigger_modelling_e2e_sqs.py

"""Enqueue one SQS message per postcode group for the modelling_e2e Lambda.

Reads postcode → property ID groups from the file produced by
list_properties_by_postcode.py, skips postcodes already logged in the tracking
markdown, then sends one SQS message per postcode batch.

Edit the CONFIG block below, then hit Run.
AWS creds come from the ambient ~/.aws profile.
"""

from __future__ import annotations

from utilities.logger import setup_logger

# ---------------------------------------------------------------------------
# CONFIG — edit these before running
# ---------------------------------------------------------------------------
PORTFOLIO_ID: int = 796
SCENARIO_ID: int = 1268
SQS_QUEUE_NAME: str = "modelling_e2e-queue-dev"

# Number of postcodes to process this run.
POSTCODES_LIMIT: int = 50

# True → Lambda runs the full pipeline but skips all DB writes (safe for testing).
DRY_RUN: bool = False

# True → Lambda skips the Google Solar fetch.
NO_SOLAR: bool = False
# ---------------------------------------------------------------------------

import ast
import json
import sys
from datetime import date
from pathlib import Path
from typing import Any, cast
from uuid import uuid4

_REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(_REPO_ROOT))

import boto3  # noqa: E402

logger = setup_logger()

_POSTCODES_FILE = _REPO_ROOT / "scripts" / f"properties_by_postcode_{PORTFOLIO_ID}.txt"
_PROCESSED_MD = _REPO_ROOT / "scripts" / f"processed_postcodes_{PORTFOLIO_ID}.md"


def _load_postcode_map() -> dict[str, list[int]]:
    if not _POSTCODES_FILE.exists():
        raise FileNotFoundError(
            f"{_POSTCODES_FILE} not found — run list_properties_by_postcode.py first"
        )
    result: dict[str, list[int]] = {}
    for line in _POSTCODES_FILE.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line or line.startswith("Total"):
            continue
        postcode_repr, ids_repr = line.split(": ", 1)
        result[ast.literal_eval(postcode_repr)] = ast.literal_eval(ids_repr)
    return result


def _already_processed() -> set[str]:
    if not _PROCESSED_MD.exists():
        return set()
    processed: set[str] = set()
    for line in _PROCESSED_MD.read_text(encoding="utf-8").splitlines():
        if (
            line.startswith("| ")
            and not line.startswith("| Postcode")
            and "---" not in line
        ):
            parts = [p.strip() for p in line.split("|")]
            if len(parts) >= 2 and parts[1]:
                processed.add(parts[1])
    return processed


def _record_processed(postcode: str, ids: list[int]) -> None:
    today = date.today().isoformat()
    ids_str = ", ".join(str(i) for i in ids)
    row = f"| {postcode} | {ids_str} | {today} |\n"
    if not _PROCESSED_MD.exists():
        _PROCESSED_MD.write_text(
            f"# Processed Properties — Portfolio {PORTFOLIO_ID}\n\n"
            "| Postcode | Property IDs | Processed |\n"
            "|----------|--------------|-----------|\n" + row,
            encoding="utf-8",
        )
    else:
        with _PROCESSED_MD.open("a", encoding="utf-8") as f:
            f.write(row)


def main() -> None:
    postcode_map = _load_postcode_map()
    done = _already_processed()

    pending = [(pc, ids) for pc, ids in postcode_map.items() if pc not in done]
    to_process = pending[:POSTCODES_LIMIT]

    if not to_process:
        logger.info("Nothing left to process.")
        return

    sqs: Any = cast(
        Any, boto3.client("sqs", region_name="eu-west-2")
    )  # pyright: ignore[reportUnknownMemberType]
    sqs_url: str = sqs.get_queue_url(QueueName=SQS_QUEUE_NAME)["QueueUrl"]

    logger.info(
        f"sending {len(to_process)} messages "
        f"(portfolio={PORTFOLIO_ID}, scenario={SCENARIO_ID}, "
        f"dry_run={DRY_RUN}, no_solar={NO_SOLAR}) → {sqs_url}"
    )

    for postcode, ids in to_process:
        sqs.send_message(
            QueueUrl=sqs_url,
            MessageBody=json.dumps(
                {
                    "property_ids": ids,
                    "portfolio_id": PORTFOLIO_ID,
                    "scenario_id": SCENARIO_ID,
                    "no_solar": NO_SOLAR,
                    "dry_run": DRY_RUN,
                }
            ),
        )
        _record_processed(postcode, ids)
        logger.info(f"  sent {postcode}: {ids}")

    logger.info(
        f"\ndone — {len(to_process)} messages enqueued, tracking → {_PROCESSED_MD}"
    )


main()