diff --git a/pytest.ini b/pytest.ini index 2bcd6178..cb6af047 100644 --- a/pytest.ini +++ b/pytest.ini @@ -25,5 +25,6 @@ testpaths = etl/epc_clean/tests etl/hubspot/tests etl/spatial/tests + tests/ markers = integration: mark a test as an integration test diff --git a/scripts/processed_postcodes_796.md b/scripts/processed_postcodes_796.md index ca563834..53a05f69 100644 --- a/scripts/processed_postcodes_796.md +++ b/scripts/processed_postcodes_796.md @@ -2,7 +2,89 @@ | Postcode | Property IDs | Processed | |----------|--------------|-----------| + | BN10 8EQ | 739489 | 2026-06-23 | | BN11 2LT | 740998 | 2026-06-23 | +| BN11 4HB | 733255 | 2026-06-23 | +| BN13 2QP | 739481 | 2026-06-23 | +| BN14 7EH | 725823 | 2026-06-23 | +| BN15 0NF | 739509 | 2026-06-23 | +| BN15 9HT | 739500 | 2026-06-23 | +| BN15 9JB | 723564 | 2026-06-23 | +| BN15 9LH | 715324 | 2026-06-23 | +| BN15 9PA | 715063 | 2026-06-23 | +| BN1 5LA | 725286 | 2026-06-23 | +| BN17 6RE | 717496 | 2026-06-23 | +| BN17 7NS | 715550 | 2026-06-23 | +| BN1 7BX | 723873 | 2026-06-23 | +| BN1 7ER | 739508 | 2026-06-23 | +| BN18 0RT | 719161 | 2026-06-23 | +| BN1 8DH | 727964 | 2026-06-23 | +| BN1 8LG | 719523 | 2026-06-23 | +| BN1 9BW | 733567 | 2026-06-23 | +| BN1 9BZ | 733592 | 2026-06-23 | +| BN2 0FB | 715445 | 2026-06-23 | +| BN2 3PE | 733597 | 2026-06-23 | +| BN2 4BE | 728215 | 2026-06-23 | +| BN2 4JS | 731660 | 2026-06-23 | +| BN2 4LL | 715371 | 2026-06-23 | +| BN2 4RT | 710129 | 2026-06-23 | +| BN2 4RW | 725006 | 2026-06-23 | +| BN2 4TJ | 726521 | 2026-06-23 | +| BN2 6TL | 727827 | 2026-06-23 | +| BN2 8LS | 725067 | 2026-06-23 | +| BN3 5AE | 729390 | 2026-06-23 | +| BN3 5AG | 726500 | 2026-06-23 | +| BN41 1DQ | 722904 | 2026-06-23 | +| BN41 1SA | 731765 | 2026-06-23 | +| BN43 5ZE | 741057 | 2026-06-23 | | BN11 2NH | 733234 | 2026-06-23 | | BN11 4EP | 730259 | 2026-06-23 | +| BN13 2NN | 739497 | 2026-06-23 | +| BN15 0DU | 739485 | 2026-06-23 | +| BN15 0LT | 731504 | 2026-06-23 | +| BN16 3HL | 719864 | 2026-06-23 | +| BN1 7EH | 726123 | 2026-06-23 | +| BN2 0AE | 715891 | 2026-06-23 | +| BN2 4PL | 716049 | 2026-06-23 | +| BN2 4TQ | 726467 | 2026-06-23 | +| BN2 9ZN | 721815 | 2026-06-23 | +| BN3 1DL | 741420 | 2026-06-23 | +| BN3 7FN | 717435 | 2026-06-23 | +| BN41 2TP | 723881 | 2026-06-23 | +| BN42 4HS | 728370 | 2026-06-23 | +| BN43 5LA | 724702 | 2026-06-23 | +| BN43 6NN | 728476 | 2026-06-23 | +| BN43 6WE | 741142 | 2026-06-23 | +| BN44 3JG | 720812 | 2026-06-23 | +| BN44 3JH | 719419 | 2026-06-23 | +| BN44 3JT | 712902 | 2026-06-23 | +| BN7 2QF | 733723 | 2026-06-23 | +| BR1 2ND | 741004 | 2026-06-23 | +| BR1 2SN | 718709 | 2026-06-23 | +| BR1 3LR | 728414 | 2026-06-23 | +| BR1 3PE | 712401 | 2026-06-23 | +| BR1 3QH | 733236 | 2026-06-23 | +| BR1 3QJ | 726048 | 2026-06-23 | +| BR1 3QW | 711383 | 2026-06-23 | +| BR1 3RG | 715615 | 2026-06-23 | +| BR1 3RW | 728883 | 2026-06-23 | +| BR1 3ST | 712335 | 2026-06-23 | +| BR1 4DE | 723589 | 2026-06-23 | +| BR1 4JZ | 730244 | 2026-06-23 | +| BR1 4NW | 726966 | 2026-06-23 | +| BR1 4PU | 717828 | 2026-06-23 | +| BR1 4PY | 711228 | 2026-06-23 | +| BR1 4QF | 710339 | 2026-06-23 | +| BR1 4QQ | 729946 | 2026-06-23 | +| BR1 4QW | 710693 | 2026-06-23 | +| BR1 4RE | 722824 | 2026-06-23 | +| BR1 4SA | 733315 | 2026-06-23 | +| BR1 4SG | 711136 | 2026-06-23 | +| BR1 4SH | 730586 | 2026-06-23 | +| BR1 5BB | 731579 | 2026-06-23 | +| BR1 5EG | 726592 | 2026-06-23 | +| BR1 5EN | 723121 | 2026-06-23 | +| BR1 5EP | 714805 | 2026-06-23 | +| BR1 5ER | 715406 | 2026-06-23 | +| BR1 5EZ | 719897 | 2026-06-23 | diff --git a/scripts/trigger_modelling_e2e_sqs.py b/scripts/trigger_modelling_e2e_sqs.py index 6091e754..6aeea292 100644 --- a/scripts/trigger_modelling_e2e_sqs.py +++ b/scripts/trigger_modelling_e2e_sqs.py @@ -10,6 +10,8 @@ AWS creds come from the ambient ~/.aws profile. from __future__ import annotations +from utilities.logger import setup_logger + # --------------------------------------------------------------------------- # CONFIG — edit these before running # --------------------------------------------------------------------------- @@ -18,10 +20,10 @@ SCENARIO_ID: int = 1268 SQS_QUEUE_NAME: str = "modelling_e2e-queue-dev" # Number of postcodes to process this run. -POSTCODES_LIMIT: int = 2 +POSTCODES_LIMIT: int = 50 # True → Lambda runs the full pipeline but skips all DB writes (safe for testing). -DRY_RUN: bool = True +DRY_RUN: bool = False # True → Lambda skips the Google Solar fetch. NO_SOLAR: bool = False @@ -40,6 +42,8 @@ sys.path.insert(0, str(_REPO_ROOT)) import boto3 # noqa: E402 +logger = setup_logger() + _POSTCODES_FILE = _REPO_ROOT / "scripts" / f"properties_by_postcode_{PORTFOLIO_ID}.txt" _PROCESSED_MD = _REPO_ROOT / "scripts" / f"processed_postcodes_{PORTFOLIO_ID}.md" @@ -99,7 +103,7 @@ def main() -> None: to_process = pending[:POSTCODES_LIMIT] if not to_process: - print("Nothing left to process.") + logger.info("Nothing left to process.") return sqs: Any = cast( @@ -107,7 +111,7 @@ def main() -> None: ) # pyright: ignore[reportUnknownMemberType] sqs_url: str = sqs.get_queue_url(QueueName=SQS_QUEUE_NAME)["QueueUrl"] - print( + logger.info( f"sending {len(to_process)} messages " f"(portfolio={PORTFOLIO_ID}, scenario={SCENARIO_ID}, " f"dry_run={DRY_RUN}, no_solar={NO_SOLAR}) → {sqs_url}" @@ -127,9 +131,11 @@ def main() -> None: ), ) _record_processed(postcode, ids) - print(f" sent {postcode}: {ids}") + logger.info(f" sent {postcode}: {ids}") - print(f"\ndone — {len(to_process)} messages enqueued, tracking → {_PROCESSED_MD}") + logger.info( + f"\ndone — {len(to_process)} messages enqueued, tracking → {_PROCESSED_MD}" + ) main()