From d33c171494bccb3d113099021775dc60a380ca65 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 09:53:42 +0000 Subject: [PATCH 01/34] =?UTF-8?q?The=20deal=20database=20gateway=20records?= =?UTF-8?q?=20a=20deal's=20OpenHousing=20job=5Fno=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- repositories/hubspot_deals/__init__.py | 0 .../deal_database_postgres_gateway.py | 22 +++++++++ tests/repositories/hubspot_deals/__init__.py | 0 .../test_deal_database_postgres_gateway.py | 48 +++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 repositories/hubspot_deals/__init__.py create mode 100644 repositories/hubspot_deals/deal_database_postgres_gateway.py create mode 100644 tests/repositories/hubspot_deals/__init__.py create mode 100644 tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py diff --git a/repositories/hubspot_deals/__init__.py b/repositories/hubspot_deals/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/repositories/hubspot_deals/deal_database_postgres_gateway.py b/repositories/hubspot_deals/deal_database_postgres_gateway.py new file mode 100644 index 000000000..56accf947 --- /dev/null +++ b/repositories/hubspot_deals/deal_database_postgres_gateway.py @@ -0,0 +1,22 @@ +"""Postgres implementation of the Abri deal-database gateway. + +First production implementation of the DealDatabaseGateway protocol used by +the AbriOrchestrator (orchestration/abri_orchestrator.py): job numbers live +on the hubspot_deal_data table, in the job_no column the scraper also +upserts from HubSpot's client_booking_reference property. +""" + +from typing import Optional + +from sqlmodel import Session + + +class DealDatabasePostgresGateway: + def __init__(self, session: Session) -> None: + self._session = session + + def record_job_no(self, deal_id: str, job_no: str) -> None: + raise NotImplementedError + + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + raise NotImplementedError diff --git a/tests/repositories/hubspot_deals/__init__.py b/tests/repositories/hubspot_deals/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py new file mode 100644 index 000000000..9e6e0c9a2 --- /dev/null +++ b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py @@ -0,0 +1,48 @@ +from collections.abc import Iterator + +import pytest +from sqlalchemy import Engine +from sqlmodel import Session + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from repositories.hubspot_deals.deal_database_postgres_gateway import ( + DealDatabasePostgresGateway, +) + +DEAL_ID = "9876543210" + + +@pytest.fixture +def session(db_engine: Engine) -> Iterator[Session]: + with Session(db_engine) as s: + yield s + + +def _insert_deal(session: Session, deal_id: str) -> HubspotDealData: + deal = HubspotDealData(deal_id=deal_id, dealname="49 Admers Crescent") + session.add(deal) + session.commit() + session.refresh(deal) + return deal + + +def test_record_job_no_persists_the_job_no_on_the_deal_row(session: Session) -> None: + # Arrange + deal = _insert_deal(session, DEAL_ID) + gateway = DealDatabasePostgresGateway(session=session) + + # Act + gateway.record_job_no(deal_id=DEAL_ID, job_no="AD0226519") + + # Assert + session.refresh(deal) + assert deal.job_no == "AD0226519" + + +def test_record_job_no_for_an_unknown_deal_raises(session: Session) -> None: + # Arrange + gateway = DealDatabasePostgresGateway(session=session) + + # Act / Assert + with pytest.raises(ValueError): + gateway.record_job_no(deal_id="0000000000", job_no="AD0226519") From 28c7e44c955d4b7bfa430878fbab4fcc8a9cdb4e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 09:54:16 +0000 Subject: [PATCH 02/34] =?UTF-8?q?The=20deal=20database=20gateway=20records?= =?UTF-8?q?=20a=20deal's=20OpenHousing=20job=5Fno=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../deal_database_postgres_gateway.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/repositories/hubspot_deals/deal_database_postgres_gateway.py b/repositories/hubspot_deals/deal_database_postgres_gateway.py index 56accf947..29c1868ec 100644 --- a/repositories/hubspot_deals/deal_database_postgres_gateway.py +++ b/repositories/hubspot_deals/deal_database_postgres_gateway.py @@ -8,7 +8,9 @@ upserts from HubSpot's client_booking_reference property. from typing import Optional -from sqlmodel import Session +from sqlmodel import Session, select + +from backend.app.db.models.hubspot_deal_data import HubspotDealData class DealDatabasePostgresGateway: @@ -16,7 +18,16 @@ class DealDatabasePostgresGateway: self._session = session def record_job_no(self, deal_id: str, job_no: str) -> None: - raise NotImplementedError + deal = self._find_deal(deal_id) + if deal is None: + raise ValueError(f"HubSpot deal {deal_id} not found") + deal.job_no = job_no + self._session.add(deal) + self._session.commit() def job_no_for_deal(self, deal_id: str) -> Optional[str]: raise NotImplementedError + + def _find_deal(self, deal_id: str) -> Optional[HubspotDealData]: + statement = select(HubspotDealData).where(HubspotDealData.deal_id == deal_id) + return self._session.exec(statement).first() From 2b29a2412b05c58441802e37afac2efb0e8645e6 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 09:54:56 +0000 Subject: [PATCH 03/34] =?UTF-8?q?The=20deal=20database=20gateway=20reads?= =?UTF-8?q?=20back=20a=20deal's=20recorded=20job=5Fno=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../test_deal_database_postgres_gateway.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py index 9e6e0c9a2..89d4cde38 100644 --- a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py +++ b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py @@ -46,3 +46,41 @@ def test_record_job_no_for_an_unknown_deal_raises(session: Session) -> None: # Act / Assert with pytest.raises(ValueError): gateway.record_job_no(deal_id="0000000000", job_no="AD0226519") + + +def test_job_no_for_deal_returns_the_recorded_job_no(session: Session) -> None: + # Arrange + _insert_deal(session, DEAL_ID) + gateway = DealDatabasePostgresGateway(session=session) + gateway.record_job_no(deal_id=DEAL_ID, job_no="AD0226519") + + # Act + job_no = gateway.job_no_for_deal(DEAL_ID) + + # Assert + assert job_no == "AD0226519" + + +def test_job_no_for_deal_returns_none_before_any_job_is_recorded( + session: Session, +) -> None: + # Arrange + _insert_deal(session, DEAL_ID) + gateway = DealDatabasePostgresGateway(session=session) + + # Act + job_no = gateway.job_no_for_deal(DEAL_ID) + + # Assert + assert job_no is None + + +def test_job_no_for_deal_returns_none_for_an_unknown_deal(session: Session) -> None: + # Arrange + gateway = DealDatabasePostgresGateway(session=session) + + # Act + job_no = gateway.job_no_for_deal("0000000000") + + # Assert + assert job_no is None From 3e11dc54e5bbda207975f88678935a35be73c2e8 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 09:55:24 +0000 Subject: [PATCH 04/34] =?UTF-8?q?The=20deal=20database=20gateway=20reads?= =?UTF-8?q?=20back=20a=20deal's=20recorded=20job=5Fno=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- repositories/hubspot_deals/deal_database_postgres_gateway.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repositories/hubspot_deals/deal_database_postgres_gateway.py b/repositories/hubspot_deals/deal_database_postgres_gateway.py index 29c1868ec..bfd917781 100644 --- a/repositories/hubspot_deals/deal_database_postgres_gateway.py +++ b/repositories/hubspot_deals/deal_database_postgres_gateway.py @@ -26,7 +26,8 @@ class DealDatabasePostgresGateway: self._session.commit() def job_no_for_deal(self, deal_id: str) -> Optional[str]: - raise NotImplementedError + deal = self._find_deal(deal_id) + return deal.job_no if deal is not None else None def _find_deal(self, deal_id: str) -> Optional[HubspotDealData]: statement = select(HubspotDealData).where(HubspotDealData.deal_id == deal_id) From a93346ec7fef2aaa1f035febf49442457d7ce8c2 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 09:56:54 +0000 Subject: [PATCH 05/34] =?UTF-8?q?A=20changed=20booking=20amends=20the=20de?= =?UTF-8?q?al's=20OpenHousing=20appointment=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- orchestration/abri_orchestrator.py | 27 +++ .../test_abri_orchestrator_amend_job.py | 180 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 tests/orchestration/test_abri_orchestrator_amend_job.py diff --git a/orchestration/abri_orchestrator.py b/orchestration/abri_orchestrator.py index 920eaffcf..3238066f4 100644 --- a/orchestration/abri_orchestrator.py +++ b/orchestration/abri_orchestrator.py @@ -5,6 +5,8 @@ from typing import Dict, List, Optional, Protocol, Tuple, Union from domain.abri.descriptions import build_job_descriptions from domain.abri.models import ( AbriRequestRejected, + AmendJobRequest, + AppointmentAmended, LogJobRequest, PlaceRef, Tenant, @@ -123,6 +125,26 @@ LogJobOrchestrationResult = Union[LogJobSummary, AbriRequestRejected] class DealDatabaseGateway(Protocol): def record_job_no(self, deal_id: str, job_no: str) -> None: ... + def job_no_for_deal(self, deal_id: str) -> Optional[str]: ... + + +AmendJobOrchestrationResult = Union[AppointmentAmended, AbriRequestRejected] + + +class JobNoNotYetRecordedError(Exception): + """An amendment arrived before the deal's job_no landed in the database. + + Retriable: the message should be redelivered until the log flow's + write-back lands (or it dead-letters), so a rapid log-then-amend + sequence eventually converges. + """ + + def __init__(self, deal_id: str) -> None: + self.deal_id = deal_id + super().__init__( + f"no job_no recorded yet for deal {deal_id}; cannot amend its appointment" + ) + class LogJobWriteBackError(Exception): """A job logged in OpenHousing whose job_no could not be written back. @@ -208,6 +230,11 @@ class AbriOrchestrator: ), ) + def amend_job( + self, booking: ConfirmedSurveyBooking + ) -> AmendJobOrchestrationResult: + raise NotImplementedError + def log_job(self, booking: ConfirmedSurveyBooking) -> LogJobOrchestrationResult: descriptions = build_job_descriptions(booking.deal_name) outcome = self._abri.log_job( diff --git a/tests/orchestration/test_abri_orchestrator_amend_job.py b/tests/orchestration/test_abri_orchestrator_amend_job.py new file mode 100644 index 000000000..44b984c2a --- /dev/null +++ b/tests/orchestration/test_abri_orchestrator_amend_job.py @@ -0,0 +1,180 @@ +import xml.etree.ElementTree as ET +from datetime import date +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from unittest.mock import MagicMock, patch + +import pytest + +from domain.abri.models import AbriRequestRejected, AppointmentAmended, PlaceRef +from infrastructure.abri.abri_client import AbriClient +from infrastructure.abri.config import AbriConfig +from infrastructure.hubspot.deal_contacts_client import HubspotDealContactsClient +from infrastructure.hubspot.deal_properties_client import HubspotDealPropertiesClient +from orchestration.abri_orchestrator import ( + AbriOrchestrator, + ConfirmedSurveyBooking, + JobNoNotYetRecordedError, +) + +FIXTURE_DIR = Path(__file__).parents[1] / "abri" +ENDPOINT_URL = "https://relay.example.test/api/DomnaRelay?code=test-function-key" +DEAL_ID = "9876543210" +JOB_NO = "AC0439951" + +CONFIG = AbriConfig( + endpoint_url=ENDPOINT_URL, + username="DomnaWeb", + password="", + default_resource="NAULKH", +) + +BOOKING = ConfirmedSurveyBooking( + deal_id=DEAL_ID, + place_ref=PlaceRef("1007165"), + deal_name="49 Admers Crescent", + confirmed_survey_date=date(2026, 6, 24), + confirmed_survey_time="14:30", +) + + +def _load_fixture(name: str) -> bytes: + return (FIXTURE_DIR / name).read_bytes() + + +class FakeDealDatabase: + """In-memory stand-in for the deal-database gateway.""" + + def __init__(self) -> None: + self.recorded_job_nos: List[Tuple[str, str]] = [] + self.job_nos: Dict[str, str] = {} + + def record_job_no(self, deal_id: str, job_no: str) -> None: + self.recorded_job_nos.append((deal_id, job_no)) + self.job_nos[deal_id] = job_no + + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + return self.job_nos.get(deal_id) + + +@pytest.fixture() +def mock_session() -> MagicMock: + return MagicMock() + + +@pytest.fixture() +def deal_database() -> FakeDealDatabase: + return FakeDealDatabase() + + +@pytest.fixture() +def orchestrator( + mock_session: MagicMock, + deal_database: FakeDealDatabase, +) -> AbriOrchestrator: + sdk_client = MagicMock() + with patch( + "infrastructure.abri.abri_client.requests.Session", + return_value=mock_session, + ): + abri_client = AbriClient(config=CONFIG) + return AbriOrchestrator( + abri_client=abri_client, + deal_contacts=HubspotDealContactsClient(sdk_client=sdk_client), + deal_properties=HubspotDealPropertiesClient(sdk_client=sdk_client), + deal_database=deal_database, + ) + + +# --- outbound request: the AmendOptiAppt envelope targets the recorded job --- + + +def test_amend_job_sends_an_amendoptiappt_envelope_for_the_deals_recorded_job( + orchestrator: AbriOrchestrator, + mock_session: MagicMock, + deal_database: FakeDealDatabase, +) -> None: + # Arrange + deal_database.job_nos[DEAL_ID] = JOB_NO + mock_session.post.return_value.content = _load_fixture( + "abri_relay_amendoptiappt_success_response.xml" + ) + + # Act + orchestrator.amend_job(BOOKING) + + # Assert + (url,) = mock_session.post.call_args.args + sent_body: bytes = mock_session.post.call_args.kwargs["data"] + sent_parameters = { + parameter.get("attribute"): parameter.get("attribute_value") + for parameter in ET.fromstring(sent_body).findall("Body/Request/Parameters") + } + assert url == ENDPOINT_URL + assert sent_parameters == { + "job_no": JOB_NO, + "action": "amend", + "appointment_date": "24/06/2026", + "appointment_time": "PM", + } + + +# --- happy path: the amended appointment is confirmed back --- + + +def test_amend_job_returns_the_amended_appointment( + orchestrator: AbriOrchestrator, + mock_session: MagicMock, + deal_database: FakeDealDatabase, +) -> None: + # Arrange + deal_database.job_nos[DEAL_ID] = JOB_NO + mock_session.post.return_value.content = _load_fixture( + "abri_relay_amendoptiappt_success_response.xml" + ) + + # Act + result = orchestrator.amend_job(BOOKING) + + # Assert + assert result == AppointmentAmended( + job_no=JOB_NO, + appointment_date="24/06/2026", + appointment_time="PM", + ) + + +# --- ordering race: amendment before the job_no write-back has landed --- + + +def test_amend_job_raises_a_retriable_error_when_no_job_no_has_been_recorded( + orchestrator: AbriOrchestrator, + mock_session: MagicMock, +) -> None: + # Act / Assert + with pytest.raises(JobNoNotYetRecordedError): + orchestrator.amend_job(BOOKING) + assert mock_session.post.call_count == 0 + + +# --- rejection passthrough: OpenHousing's actual reason, verbatim --- + + +def test_amend_job_returns_the_openhousing_rejection_verbatim( + orchestrator: AbriOrchestrator, + mock_session: MagicMock, + deal_database: FakeDealDatabase, +) -> None: + # Arrange + deal_database.job_nos[DEAL_ID] = JOB_NO + mock_session.post.return_value.content = _load_fixture( + "abri_relay_amendoptiappt_relayerror_response.xml" + ) + + # Act + result = orchestrator.amend_job(BOOKING) + + # Assert + assert isinstance(result, AbriRequestRejected) + assert result.code == "RelayError" + assert result.message.startswith("Job_no 1019905 is invalid") From 9bae8df6ea913cda19b865332b6b4fcac1b00b75 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:01:01 +0000 Subject: [PATCH 06/34] =?UTF-8?q?A=20changed=20booking=20amends=20the=20de?= =?UTF-8?q?al's=20OpenHousing=20appointment=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- orchestration/abri_orchestrator.py | 14 +++++++++++++- scripts/smoke_test_tenant_contacts.py | 7 +++++-- .../test_abri_orchestrator_log_job.py | 12 +++++++++++- .../test_abri_orchestrator_tenant_data_sync.py | 4 ++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/orchestration/abri_orchestrator.py b/orchestration/abri_orchestrator.py index 3238066f4..397bc9d77 100644 --- a/orchestration/abri_orchestrator.py +++ b/orchestration/abri_orchestrator.py @@ -233,7 +233,19 @@ class AbriOrchestrator: def amend_job( self, booking: ConfirmedSurveyBooking ) -> AmendJobOrchestrationResult: - raise NotImplementedError + job_no = self._deal_database.job_no_for_deal(booking.deal_id) + if job_no is None: + raise JobNoNotYetRecordedError(deal_id=booking.deal_id) + + return self._abri.amend_job( + AmendJobRequest( + job_no=job_no, + appointment_date=booking.confirmed_survey_date, + appointment_time=slot_for_confirmed_time( + booking.confirmed_survey_time + ), + ) + ) def log_job(self, booking: ConfirmedSurveyBooking) -> LogJobOrchestrationResult: descriptions = build_job_descriptions(booking.deal_name) diff --git a/scripts/smoke_test_tenant_contacts.py b/scripts/smoke_test_tenant_contacts.py index f9b55dc3b..8dedeedac 100644 --- a/scripts/smoke_test_tenant_contacts.py +++ b/scripts/smoke_test_tenant_contacts.py @@ -22,7 +22,7 @@ All tenant details below are fictional (Ofcom-reserved number ranges). import os from pathlib import Path -from typing import Any, List, cast +from typing import Any, List, Optional, cast from dotenv import dotenv_values from hubspot.client import Client # type: ignore[reportMissingTypeStubs] @@ -101,11 +101,14 @@ def _stubbed_abri_client() -> AbriClient: class _UnusedDealDatabase: - """Satisfies the gateway dependency; the tenant-data flow never writes it.""" + """Satisfies the gateway dependency; the tenant-data flow never touches it.""" def record_job_no(self, deal_id: str, job_no: str) -> None: raise AssertionError("tenant-data sync must not touch the deal database") + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + raise AssertionError("tenant-data sync must not touch the deal database") + def _archive_contacts(sdk_client: Client) -> None: if not CONTACT_IDS_TO_ARCHIVE: diff --git a/tests/orchestration/test_abri_orchestrator_log_job.py b/tests/orchestration/test_abri_orchestrator_log_job.py index b91739c52..c47a7fcd2 100644 --- a/tests/orchestration/test_abri_orchestrator_log_job.py +++ b/tests/orchestration/test_abri_orchestrator_log_job.py @@ -3,7 +3,7 @@ import traceback import xml.etree.ElementTree as ET from datetime import date from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple from unittest.mock import MagicMock, patch import pytest @@ -57,6 +57,16 @@ class FakeDealDatabase: def record_job_no(self, deal_id: str, job_no: str) -> None: self.recorded_job_nos.append((deal_id, job_no)) + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + return next( + ( + job_no + for recorded_deal_id, job_no in self.recorded_job_nos + if recorded_deal_id == deal_id + ), + None, + ) + @pytest.fixture() def mock_session() -> MagicMock: diff --git a/tests/orchestration/test_abri_orchestrator_tenant_data_sync.py b/tests/orchestration/test_abri_orchestrator_tenant_data_sync.py index e45e48686..58b82dd82 100644 --- a/tests/orchestration/test_abri_orchestrator_tenant_data_sync.py +++ b/tests/orchestration/test_abri_orchestrator_tenant_data_sync.py @@ -2,6 +2,7 @@ import json import traceback import xml.etree.ElementTree as ET from pathlib import Path +from typing import Optional from unittest.mock import MagicMock, patch import pytest @@ -66,6 +67,9 @@ class FakeDealDatabase: def record_job_no(self, deal_id: str, job_no: str) -> None: raise AssertionError("tenant-data sync must not touch the deal database") + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + raise AssertionError("tenant-data sync must not touch the deal database") + @pytest.fixture() def orchestrator( From fac9dde77ed048319d8974e6ac55f8b0b9825e28 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:02:45 +0000 Subject: [PATCH 07/34] =?UTF-8?q?A=20non-retriable=20failure=20marks=20the?= =?UTF-8?q?=20task=20failed=20without=20requeueing=20the=20message=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../utilities/aws_lambda/test_task_handler.py | 32 ++++++++++++++++++- utilities/aws_lambda/task_handler.py | 11 +++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/utilities/aws_lambda/test_task_handler.py b/tests/utilities/aws_lambda/test_task_handler.py index 418669bcd..9c538bce7 100644 --- a/tests/utilities/aws_lambda/test_task_handler.py +++ b/tests/utilities/aws_lambda/test_task_handler.py @@ -8,11 +8,12 @@ import pytest from sqlalchemy import Engine from sqlmodel import Session +from domain.tasks.subtasks import SubTaskStatus from domain.tasks.tasks import Source from orchestration.task_orchestrator import TaskOrchestrator from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository from repositories.tasks.task_postgres_repository import TaskPostgresRepository -from utilities.aws_lambda.task_handler import task_handler +from utilities.aws_lambda.task_handler import NonRetriableTaskError, task_handler @dataclass @@ -104,6 +105,35 @@ def test_task_handler_passes_orchestrator_and_task_id_when_flag_is_true( assert recv_task_id == UUID(result[0]["task_id"]) +def test_task_handler_does_not_requeue_a_record_failing_non_retriably( + harness: Harness, +) -> None: + # Arrange + @task_handler( + task_source="abri", + source=Source.HUBSPOT_DEAL, + orchestrator_cm=harness.factory, + ) + def handler(body: dict[str, Any], context: Any) -> None: + raise NonRetriableTaskError("job logged but write-back failed") + + event = { + "Records": [ + {"messageId": "msg-1", "body": '{"hubspot_deal_id": "123"}'} + ] + } + + # Act + result = handler(event, context=None) + + # Assert: the failure is recorded, but the message is not requeued. + assert result["batchItemFailures"] == [] + subtask_id = result["tasks"][0]["subtask_id"] + failed = harness.subtasks.get(UUID(subtask_id)) + assert failed.status is SubTaskStatus.FAILED + assert failed.outputs == {"error": "job logged but write-back failed"} + + def test_task_handler_leaves_cloudwatch_url_unset_outside_lambda( harness: Harness, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/utilities/aws_lambda/task_handler.py b/utilities/aws_lambda/task_handler.py index f2064f573..bbcf3f5cd 100644 --- a/utilities/aws_lambda/task_handler.py +++ b/utilities/aws_lambda/task_handler.py @@ -20,6 +20,17 @@ logger = logging.getLogger(__name__) OrchestratorCM = Callable[[], AbstractContextManager[TaskOrchestrator]] +class NonRetriableTaskError(Exception): + """A failure that must not be retried by SQS redelivery. + + The SubTask is marked failed (the failure record is the visibility + surface) but the message is not returned to the queue, so the work is + never re-run. Raise it for failures where a retry could duplicate an + irreversible side effect — e.g. a job already logged in OpenHousing + whose HubSpot write-back failed. + """ + + def task_handler( *, task_source: str, From 6a580d7691531edb2858d6d634c1ce21c2ceec48 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:03:36 +0000 Subject: [PATCH 08/34] =?UTF-8?q?A=20non-retriable=20failure=20marks=20the?= =?UTF-8?q?=20task=20failed=20without=20requeueing=20the=20message=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../utilities/aws_lambda/test_task_handler.py | 27 +++++++++++++++++++ utilities/aws_lambda/task_handler.py | 11 ++++++++ 2 files changed, 38 insertions(+) diff --git a/tests/utilities/aws_lambda/test_task_handler.py b/tests/utilities/aws_lambda/test_task_handler.py index 9c538bce7..10d82979d 100644 --- a/tests/utilities/aws_lambda/test_task_handler.py +++ b/tests/utilities/aws_lambda/test_task_handler.py @@ -105,6 +105,33 @@ def test_task_handler_passes_orchestrator_and_task_id_when_flag_is_true( assert recv_task_id == UUID(result[0]["task_id"]) +def test_task_handler_reports_an_ordinarily_failing_record_for_redelivery( + harness: Harness, +) -> None: + # Arrange + @task_handler( + task_source="abri", + source=Source.HUBSPOT_DEAL, + orchestrator_cm=harness.factory, + ) + def handler(body: dict[str, Any], context: Any) -> None: + raise RuntimeError("transient failure") + + event = { + "Records": [ + {"messageId": "msg-1", "body": '{"hubspot_deal_id": "123"}'} + ] + } + + # Act + result = handler(event, context=None) + + # Assert + assert result["batchItemFailures"] == [{"itemIdentifier": "msg-1"}] + subtask_id = result["tasks"][0]["subtask_id"] + assert harness.subtasks.get(UUID(subtask_id)).status is SubTaskStatus.FAILED + + def test_task_handler_does_not_requeue_a_record_failing_non_retriably( harness: Harness, ) -> None: diff --git a/utilities/aws_lambda/task_handler.py b/utilities/aws_lambda/task_handler.py index bbcf3f5cd..0298f3814 100644 --- a/utilities/aws_lambda/task_handler.py +++ b/utilities/aws_lambda/task_handler.py @@ -89,6 +89,17 @@ def task_handler( work=lambda: func(body, context), cloud_logs_url=cloud_logs_url, ) + except NonRetriableTaskError: + # Failed, but never redelivered: the SubTask's failure + # record is the only follow-up surface. + logger.exception( + "subtask failed non-retriably " + "(task_source=%s source_id=%s)", + task_source, + source_id, + ) + if "Records" not in event: + raise except Exception: logger.exception( "subtask failed (task_source=%s source_id=%s)", From c2814561646ef9e05ec26553a3420e39de728655 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:04:55 +0000 Subject: [PATCH 09/34] =?UTF-8?q?A=20trigger=20message=20validates=20the?= =?UTF-8?q?=20fields=20each=20fired=20Abri=20flow=20needs=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- applications/abri/__init__.py | 0 applications/abri/abri_trigger_request.py | 28 +++++ tests/applications/abri/__init__.py | 0 .../abri/test_abri_trigger_request.py | 103 ++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 applications/abri/__init__.py create mode 100644 applications/abri/abri_trigger_request.py create mode 100644 tests/applications/abri/__init__.py create mode 100644 tests/applications/abri/test_abri_trigger_request.py diff --git a/applications/abri/__init__.py b/applications/abri/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/applications/abri/abri_trigger_request.py b/applications/abri/abri_trigger_request.py new file mode 100644 index 000000000..d159105c7 --- /dev/null +++ b/applications/abri/abri_trigger_request.py @@ -0,0 +1,28 @@ +"""The message contract between the HubSpot scraper and the abri lambda. + +One message per scrape, naming every Abri flow that fired plus the deal +fields those flows need. A message missing a field its flow needs fails +validation — the resulting failed task / DLQ entry is the agreed +visibility surface. +""" + +from datetime import date +from typing import Any, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict + +AbriFlow = Literal["amend_job", "log_job", "sync_tenant_data"] + + +class AbriTriggerRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + hubspot_deal_id: str + flows: List[AbriFlow] + place_ref: Optional[str] = None + deal_name: Optional[str] = None + confirmed_survey_date: Optional[date] = None + confirmed_survey_time: Optional[str] = None + + def model_post_init(self, __context: Any) -> None: + raise NotImplementedError diff --git a/tests/applications/abri/__init__.py b/tests/applications/abri/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/applications/abri/test_abri_trigger_request.py b/tests/applications/abri/test_abri_trigger_request.py new file mode 100644 index 000000000..9e9672a2b --- /dev/null +++ b/tests/applications/abri/test_abri_trigger_request.py @@ -0,0 +1,103 @@ +from datetime import date +from typing import Any, Dict + +import pytest +from pydantic import ValidationError + +from applications.abri.abri_trigger_request import AbriTriggerRequest + + +def _full_message() -> Dict[str, Any]: + return { + "hubspot_deal_id": "9876543210", + "flows": ["amend_job", "log_job", "sync_tenant_data"], + "place_ref": "1007165", + "deal_name": "49 Admers Crescent", + "confirmed_survey_date": "2026-06-24", + "confirmed_survey_time": "14:30", + } + + +def test_a_full_message_parses_into_a_typed_request() -> None: + # Act + request = AbriTriggerRequest.model_validate(_full_message()) + + # Assert + assert request.hubspot_deal_id == "9876543210" + assert request.flows == ["amend_job", "log_job", "sync_tenant_data"] + assert request.place_ref == "1007165" + assert request.deal_name == "49 Admers Crescent" + assert request.confirmed_survey_date == date(2026, 6, 24) + assert request.confirmed_survey_time == "14:30" + + +@pytest.mark.parametrize( + ("flow", "missing_field"), + [ + ("log_job", "place_ref"), + ("log_job", "deal_name"), + ("log_job", "confirmed_survey_date"), + ("amend_job", "confirmed_survey_date"), + ("sync_tenant_data", "place_ref"), + ], +) +def test_a_message_missing_a_field_its_flow_needs_fails_validation( + flow: str, missing_field: str +) -> None: + # Arrange + message = _full_message() + message["flows"] = [flow] + del message[missing_field] + + # Act / Assert + with pytest.raises(ValidationError, match=missing_field): + AbriTriggerRequest.model_validate(message) + + +def test_a_message_missing_only_fields_of_unfired_flows_parses() -> None: + # Arrange: tenant sync needs only the place_ref (and the deal id). + message = { + "hubspot_deal_id": "9876543210", + "flows": ["sync_tenant_data"], + "place_ref": "1007165", + } + + # Act + request = AbriTriggerRequest.model_validate(message) + + # Assert + assert request.flows == ["sync_tenant_data"] + assert request.confirmed_survey_date is None + + +def test_a_message_without_a_confirmed_time_parses_for_logging() -> None: + # Arrange: the time slot is optional — no time means an all-day slot. + message = _full_message() + message["flows"] = ["log_job"] + del message["confirmed_survey_time"] + + # Act + request = AbriTriggerRequest.model_validate(message) + + # Assert + assert request.confirmed_survey_time is None + + +def test_a_message_naming_an_unknown_flow_fails_validation() -> None: + # Arrange + message = _full_message() + message["flows"] = ["cancel_job"] + + # Act / Assert + with pytest.raises(ValidationError): + AbriTriggerRequest.model_validate(message) + + +def test_a_message_naming_no_flows_fails_validation() -> None: + # Arrange + message = _full_message() + message["flows"] = [] + + # Act / Assert + with pytest.raises(ValidationError): + AbriTriggerRequest.model_validate(message) From 8daaa0974d900593a73acb278f01e2eb847d044d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:05:25 +0000 Subject: [PATCH 10/34] =?UTF-8?q?A=20trigger=20message=20validates=20the?= =?UTF-8?q?=20fields=20each=20fired=20Abri=20flow=20needs=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- applications/abri/abri_trigger_request.py | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/applications/abri/abri_trigger_request.py b/applications/abri/abri_trigger_request.py index d159105c7..0a9346ca9 100644 --- a/applications/abri/abri_trigger_request.py +++ b/applications/abri/abri_trigger_request.py @@ -7,22 +7,38 @@ visibility surface. """ from datetime import date -from typing import Any, List, Literal, Optional +from typing import Dict, List, Literal, Optional, Tuple -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field, model_validator AbriFlow = Literal["amend_job", "log_job", "sync_tenant_data"] +# The deal fields each flow reads; hubspot_deal_id is always required. +_FIELDS_BY_FLOW: Dict[AbriFlow, Tuple[str, ...]] = { + "amend_job": ("confirmed_survey_date",), + "log_job": ("place_ref", "deal_name", "confirmed_survey_date"), + "sync_tenant_data": ("place_ref",), +} + class AbriTriggerRequest(BaseModel): model_config = ConfigDict(extra="ignore") hubspot_deal_id: str - flows: List[AbriFlow] + flows: List[AbriFlow] = Field(min_length=1) place_ref: Optional[str] = None deal_name: Optional[str] = None confirmed_survey_date: Optional[date] = None confirmed_survey_time: Optional[str] = None - def model_post_init(self, __context: Any) -> None: - raise NotImplementedError + @model_validator(mode="after") + def _each_fired_flow_has_its_fields(self) -> "AbriTriggerRequest": + missing = [ + f"flow {flow} requires {field}" + for flow in self.flows + for field in _FIELDS_BY_FLOW[flow] + if getattr(self, field) is None + ] + if missing: + raise ValueError("; ".join(missing)) + return self From 8d70679ab0c9e99a52e5663be47ac71c53108f9f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:07:23 +0000 Subject: [PATCH 11/34] =?UTF-8?q?Amending=20an=20appointment=20needs=20onl?= =?UTF-8?q?y=20the=20deal's=20changed=20date=20and=20time=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- orchestration/abri_orchestrator.py | 21 ++++++++++++------- .../test_abri_orchestrator_amend_job.py | 16 +++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/orchestration/abri_orchestrator.py b/orchestration/abri_orchestrator.py index 397bc9d77..d4b61b8ba 100644 --- a/orchestration/abri_orchestrator.py +++ b/orchestration/abri_orchestrator.py @@ -128,6 +128,15 @@ class DealDatabaseGateway(Protocol): def job_no_for_deal(self, deal_id: str) -> Optional[str]: ... +@dataclass(frozen=True) +class AppointmentChange: + """The deal fields a changed survey booking contributes to an AmendJob.""" + + deal_id: str + confirmed_survey_date: date + confirmed_survey_time: Optional[str] + + AmendJobOrchestrationResult = Union[AppointmentAmended, AbriRequestRejected] @@ -230,19 +239,17 @@ class AbriOrchestrator: ), ) - def amend_job( - self, booking: ConfirmedSurveyBooking - ) -> AmendJobOrchestrationResult: - job_no = self._deal_database.job_no_for_deal(booking.deal_id) + def amend_job(self, change: AppointmentChange) -> AmendJobOrchestrationResult: + job_no = self._deal_database.job_no_for_deal(change.deal_id) if job_no is None: - raise JobNoNotYetRecordedError(deal_id=booking.deal_id) + raise JobNoNotYetRecordedError(deal_id=change.deal_id) return self._abri.amend_job( AmendJobRequest( job_no=job_no, - appointment_date=booking.confirmed_survey_date, + appointment_date=change.confirmed_survey_date, appointment_time=slot_for_confirmed_time( - booking.confirmed_survey_time + change.confirmed_survey_time ), ) ) diff --git a/tests/orchestration/test_abri_orchestrator_amend_job.py b/tests/orchestration/test_abri_orchestrator_amend_job.py index 44b984c2a..f5ba20e57 100644 --- a/tests/orchestration/test_abri_orchestrator_amend_job.py +++ b/tests/orchestration/test_abri_orchestrator_amend_job.py @@ -6,14 +6,14 @@ from unittest.mock import MagicMock, patch import pytest -from domain.abri.models import AbriRequestRejected, AppointmentAmended, PlaceRef +from domain.abri.models import AbriRequestRejected, AppointmentAmended from infrastructure.abri.abri_client import AbriClient from infrastructure.abri.config import AbriConfig from infrastructure.hubspot.deal_contacts_client import HubspotDealContactsClient from infrastructure.hubspot.deal_properties_client import HubspotDealPropertiesClient from orchestration.abri_orchestrator import ( AbriOrchestrator, - ConfirmedSurveyBooking, + AppointmentChange, JobNoNotYetRecordedError, ) @@ -29,10 +29,8 @@ CONFIG = AbriConfig( default_resource="NAULKH", ) -BOOKING = ConfirmedSurveyBooking( +CHANGE = AppointmentChange( deal_id=DEAL_ID, - place_ref=PlaceRef("1007165"), - deal_name="49 Admers Crescent", confirmed_survey_date=date(2026, 6, 24), confirmed_survey_time="14:30", ) @@ -101,7 +99,7 @@ def test_amend_job_sends_an_amendoptiappt_envelope_for_the_deals_recorded_job( ) # Act - orchestrator.amend_job(BOOKING) + orchestrator.amend_job(CHANGE) # Assert (url,) = mock_session.post.call_args.args @@ -134,7 +132,7 @@ def test_amend_job_returns_the_amended_appointment( ) # Act - result = orchestrator.amend_job(BOOKING) + result = orchestrator.amend_job(CHANGE) # Assert assert result == AppointmentAmended( @@ -153,7 +151,7 @@ def test_amend_job_raises_a_retriable_error_when_no_job_no_has_been_recorded( ) -> None: # Act / Assert with pytest.raises(JobNoNotYetRecordedError): - orchestrator.amend_job(BOOKING) + orchestrator.amend_job(CHANGE) assert mock_session.post.call_count == 0 @@ -172,7 +170,7 @@ def test_amend_job_returns_the_openhousing_rejection_verbatim( ) # Act - result = orchestrator.amend_job(BOOKING) + result = orchestrator.amend_job(CHANGE) # Assert assert isinstance(result, AbriRequestRejected) From 1c5a43df56645f7376564ba5bee29ed4f032ffbf Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:09:20 +0000 Subject: [PATCH 12/34] =?UTF-8?q?One=20message=20dispatches=20the=20fired?= =?UTF-8?q?=20Abri=20flows=20in=20a=20fixed,=20retry-safe=20order=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- applications/abri/dispatch.py | 59 ++++++ tests/applications/abri/test_dispatch.py | 254 +++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 applications/abri/dispatch.py create mode 100644 tests/applications/abri/test_dispatch.py diff --git a/applications/abri/dispatch.py b/applications/abri/dispatch.py new file mode 100644 index 000000000..aef12a3b2 --- /dev/null +++ b/applications/abri/dispatch.py @@ -0,0 +1,59 @@ +"""Dispatches the flows named in a trigger message through the orchestrator. + +The scraper decides which flows fire; this is a dumb dispatcher. Flows run +in a fixed, retry-safe order — amend, then log, then tenant sync — so that +redelivery re-runs idempotent flows before non-idempotent ones: amend is +naturally idempotent, log is guarded by the job-number check, and tenant +contact creation (not idempotent) runs last so its failure can never cause +a duplicate job log, and a log failure can never duplicate contacts. +""" + +from typing import Dict, Protocol + +from applications.abri.abri_trigger_request import AbriTriggerRequest +from domain.abri.models import AbriRequestRejected, PlaceRef +from orchestration.abri_orchestrator import ( + AmendJobOrchestrationResult, + AppointmentChange, + ConfirmedSurveyBooking, + DealDatabaseGateway, + LogJobOrchestrationResult, + TenantDataSyncResult, +) + + +class AbriFlows(Protocol): + """The per-flow surface of the AbriOrchestrator that dispatch drives.""" + + def amend_job(self, change: AppointmentChange) -> AmendJobOrchestrationResult: ... + + def log_job(self, booking: ConfirmedSurveyBooking) -> LogJobOrchestrationResult: ... + + def sync_tenant_data( + self, place_ref: PlaceRef, deal_id: str + ) -> TenantDataSyncResult: ... + + +class AbriFlowRejectedError(Exception): + """An OpenHousing rejection, surfaced as a failure so the task fails. + + Retrying a permanent rejection is wasteful but harmless — every flow is + safe to re-run — and redelivery eventually parks the message in the + dead-letter queue, which is the ops inbox. + """ + + def __init__(self, flow: str, rejection: AbriRequestRejected) -> None: + self.flow = flow + self.rejection = rejection + super().__init__( + f"{flow} rejected by OpenHousing: {rejection.code}: {rejection.message}" + ) + + +def dispatch_abri_flows( + request: AbriTriggerRequest, + flows: AbriFlows, + deal_database: DealDatabaseGateway, +) -> Dict[str, str]: + """Run the fired flows in the fixed order; returns a PII-free summary.""" + raise NotImplementedError diff --git a/tests/applications/abri/test_dispatch.py b/tests/applications/abri/test_dispatch.py new file mode 100644 index 000000000..b41aa4d4b --- /dev/null +++ b/tests/applications/abri/test_dispatch.py @@ -0,0 +1,254 @@ +from datetime import date +from typing import Any, Dict, List, Optional + +import pytest + +from applications.abri.abri_trigger_request import AbriTriggerRequest +from applications.abri.dispatch import AbriFlowRejectedError, dispatch_abri_flows +from domain.abri.models import AbriRequestRejected, AppointmentAmended, PlaceRef +from orchestration.abri_orchestrator import ( + AmendJobOrchestrationResult, + AppointmentChange, + ConfirmedSurveyBooking, + LogJobOrchestrationResult, + LogJobSummary, + LogJobWriteBackError, + TenantDataSyncResult, + TenantDataSyncSummary, +) +from utilities.aws_lambda.task_handler import NonRetriableTaskError + +DEAL_ID = "9876543210" + + +def _request(flows: List[str]) -> AbriTriggerRequest: + return AbriTriggerRequest.model_validate( + { + "hubspot_deal_id": DEAL_ID, + "flows": flows, + "place_ref": "1007165", + "deal_name": "49 Admers Crescent", + "confirmed_survey_date": "2026-06-24", + "confirmed_survey_time": "14:30", + } + ) + + +class FakeOrchestrator: + """Records flow invocations in order; outcomes are programmable.""" + + def __init__(self) -> None: + self.calls: List[Any] = [] + self.amend_outcome: AmendJobOrchestrationResult = AppointmentAmended( + job_no="AC0439951", + appointment_date="24/06/2026", + appointment_time="PM", + ) + self.log_outcome: LogJobOrchestrationResult = LogJobSummary( + deal_id=DEAL_ID, job_no="AD0226519" + ) + self.log_error: Optional[Exception] = None + self.sync_outcome: TenantDataSyncResult = TenantDataSyncSummary( + tenancy_reference="TEN0001", + contact_ids=("101", "102"), + vulnerable_contact_count=1, + ) + + def amend_job(self, change: AppointmentChange) -> AmendJobOrchestrationResult: + self.calls.append(("amend_job", change)) + return self.amend_outcome + + def log_job(self, booking: ConfirmedSurveyBooking) -> LogJobOrchestrationResult: + self.calls.append(("log_job", booking)) + if self.log_error is not None: + raise self.log_error + return self.log_outcome + + def sync_tenant_data( + self, place_ref: PlaceRef, deal_id: str + ) -> TenantDataSyncResult: + self.calls.append(("sync_tenant_data", place_ref, deal_id)) + return self.sync_outcome + + +class FakeDealDatabase: + """In-memory stand-in for the deal-database gateway.""" + + def __init__(self) -> None: + self.job_nos: Dict[str, str] = {} + + def record_job_no(self, deal_id: str, job_no: str) -> None: + self.job_nos[deal_id] = job_no + + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + return self.job_nos.get(deal_id) + + +@pytest.fixture() +def orchestrator() -> FakeOrchestrator: + return FakeOrchestrator() + + +@pytest.fixture() +def deal_database() -> FakeDealDatabase: + return FakeDealDatabase() + + +# --- fixed dispatch order: amend -> log -> tenant sync --- + + +def test_fired_flows_run_in_the_fixed_order_regardless_of_message_order( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["sync_tenant_data", "log_job", "amend_job"]) + + # Act + dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + assert [call[0] for call in orchestrator.calls] == [ + "amend_job", + "log_job", + "sync_tenant_data", + ] + + +def test_flows_the_message_does_not_name_do_not_run( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["sync_tenant_data"]) + + # Act + dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + assert orchestrator.calls == [ + ("sync_tenant_data", PlaceRef("1007165"), DEAL_ID) + ] + + +# --- the flows receive the deal fields from the message --- + + +def test_flows_receive_the_deal_fields_carried_by_the_message( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["amend_job", "log_job"]) + + # Act + dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + amend_call, log_call = orchestrator.calls + assert amend_call == ( + "amend_job", + AppointmentChange( + deal_id=DEAL_ID, + confirmed_survey_date=date(2026, 6, 24), + confirmed_survey_time="14:30", + ), + ) + assert log_call == ( + "log_job", + ConfirmedSurveyBooking( + deal_id=DEAL_ID, + place_ref=PlaceRef("1007165"), + deal_name="49 Admers Crescent", + confirmed_survey_date=date(2026, 6, 24), + confirmed_survey_time="14:30", + ), + ) + + +# --- idempotency guard: a job is never logged twice for the same deal --- + + +def test_the_log_flow_is_skipped_when_the_deal_already_has_a_job_no( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["log_job", "sync_tenant_data"]) + deal_database.job_nos[DEAL_ID] = "AD0226519" + + # Act + dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + assert [call[0] for call in orchestrator.calls] == ["sync_tenant_data"] + + +def test_the_log_flow_runs_when_the_deal_has_no_job_no( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["log_job"]) + + # Act + dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + assert [call[0] for call in orchestrator.calls] == ["log_job"] + + +# --- rejections raise, so the task fails and the DLQ is the ops inbox --- + + +def test_an_openhousing_rejection_raises_and_stops_the_dispatch( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["amend_job", "sync_tenant_data"]) + orchestrator.amend_outcome = AbriRequestRejected( + code="RelayError", message="Job_no 1019905 is invalid. Job not found." + ) + + # Act / Assert + with pytest.raises(AbriFlowRejectedError) as exc_info: + dispatch_abri_flows(request, orchestrator, deal_database) + assert "RelayError" in str(exc_info.value) + assert "Job_no 1019905 is invalid" in str(exc_info.value) + assert [call[0] for call in orchestrator.calls] == ["amend_job"] + + +# --- write-back failure: failed task, but the message is never requeued --- + + +def test_a_log_write_back_failure_raises_a_non_retriable_error( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["log_job", "sync_tenant_data"]) + orchestrator.log_error = LogJobWriteBackError( + message="HubSpot update failed", deal_id=DEAL_ID, job_no="AD0226519" + ) + + # Act / Assert: the orphaned job_no stays visible in the failure record, + # and the tenant sync never runs on this delivery. + with pytest.raises(NonRetriableTaskError) as exc_info: + dispatch_abri_flows(request, orchestrator, deal_database) + assert "AD0226519" in str(exc_info.value) + assert [call[0] for call in orchestrator.calls] == ["log_job"] + + +# --- the task output is a PII-free summary of what ran --- + + +def test_dispatch_returns_a_pii_free_summary_of_what_ran( + orchestrator: FakeOrchestrator, deal_database: FakeDealDatabase +) -> None: + # Arrange + request = _request(["amend_job", "log_job", "sync_tenant_data"]) + deal_database.job_nos[DEAL_ID] = "AC0439951" + + # Act + summary = dispatch_abri_flows(request, orchestrator, deal_database) + + # Assert + assert summary == { + "amend_job": "appointment amended to 24/06/2026 PM (job AC0439951)", + "log_job": "skipped: job AC0439951 already logged", + "sync_tenant_data": "2 contacts created (1 vulnerable)", + } From a33dd43271e0a366be3c003028ddf5ed2e720402 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:10:30 +0000 Subject: [PATCH 13/34] =?UTF-8?q?One=20message=20dispatches=20the=20fired?= =?UTF-8?q?=20Abri=20flows=20in=20a=20fixed,=20retry-safe=20order=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- applications/abri/dispatch.py | 80 ++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/applications/abri/dispatch.py b/applications/abri/dispatch.py index aef12a3b2..f69981162 100644 --- a/applications/abri/dispatch.py +++ b/applications/abri/dispatch.py @@ -18,8 +18,10 @@ from orchestration.abri_orchestrator import ( ConfirmedSurveyBooking, DealDatabaseGateway, LogJobOrchestrationResult, + LogJobWriteBackError, TenantDataSyncResult, ) +from utilities.aws_lambda.task_handler import NonRetriableTaskError class AbriFlows(Protocol): @@ -56,4 +58,80 @@ def dispatch_abri_flows( deal_database: DealDatabaseGateway, ) -> Dict[str, str]: """Run the fired flows in the fixed order; returns a PII-free summary.""" - raise NotImplementedError + summary: Dict[str, str] = {} + + if "amend_job" in request.flows: + summary["amend_job"] = _run_amend(request, flows) + if "log_job" in request.flows: + summary["log_job"] = _run_log(request, flows, deal_database) + if "sync_tenant_data" in request.flows: + summary["sync_tenant_data"] = _run_tenant_sync(request, flows) + + return summary + + +def _run_amend(request: AbriTriggerRequest, flows: AbriFlows) -> str: + if request.confirmed_survey_date is None: + raise ValueError("amend_job fired without a confirmed_survey_date") + outcome = flows.amend_job( + AppointmentChange( + deal_id=request.hubspot_deal_id, + confirmed_survey_date=request.confirmed_survey_date, + confirmed_survey_time=request.confirmed_survey_time, + ) + ) + if isinstance(outcome, AbriRequestRejected): + raise AbriFlowRejectedError(flow="amend_job", rejection=outcome) + return ( + f"appointment amended to {outcome.appointment_date} " + f"{outcome.appointment_time} (job {outcome.job_no})" + ) + + +def _run_log( + request: AbriTriggerRequest, + flows: AbriFlows, + deal_database: DealDatabaseGateway, +) -> str: + existing_job_no = deal_database.job_no_for_deal(request.hubspot_deal_id) + if existing_job_no is not None: + return f"skipped: job {existing_job_no} already logged" + + if ( + request.place_ref is None + or request.deal_name is None + or request.confirmed_survey_date is None + ): + raise ValueError("log_job fired without place_ref/deal_name/survey date") + try: + outcome = flows.log_job( + ConfirmedSurveyBooking( + deal_id=request.hubspot_deal_id, + place_ref=PlaceRef(request.place_ref), + deal_name=request.deal_name, + confirmed_survey_date=request.confirmed_survey_date, + confirmed_survey_time=request.confirmed_survey_time, + ) + ) + except LogJobWriteBackError as error: + # The job exists in OpenHousing; a redelivery would log a duplicate. + # The orphaned job_no stays visible in the failure record. + raise NonRetriableTaskError(str(error)) from error + if isinstance(outcome, AbriRequestRejected): + raise AbriFlowRejectedError(flow="log_job", rejection=outcome) + return f"job {outcome.job_no} logged" + + +def _run_tenant_sync(request: AbriTriggerRequest, flows: AbriFlows) -> str: + if request.place_ref is None: + raise ValueError("sync_tenant_data fired without a place_ref") + outcome = flows.sync_tenant_data( + place_ref=PlaceRef(request.place_ref), + deal_id=request.hubspot_deal_id, + ) + if isinstance(outcome, AbriRequestRejected): + raise AbriFlowRejectedError(flow="sync_tenant_data", rejection=outcome) + return ( + f"{len(outcome.contact_ids)} contacts created " + f"({outcome.vulnerable_contact_count} vulnerable)" + ) From 2e72f32d5244ef02c3bf4da478fe201e2f6bdee1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:12:41 +0000 Subject: [PATCH 14/34] =?UTF-8?q?A=20scrape=20that=20fires=20Abri=20trigge?= =?UTF-8?q?rs=20builds=20one=20message=20naming=20the=20fired=20flows=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/abri_flow_triggers.py | 29 +++ etl/hubspot/tests/test_abri_flow_triggers.py | 236 +++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 etl/hubspot/abri_flow_triggers.py create mode 100644 etl/hubspot/tests/test_abri_flow_triggers.py diff --git a/etl/hubspot/abri_flow_triggers.py b/etl/hubspot/abri_flow_triggers.py new file mode 100644 index 000000000..a379acd6f --- /dev/null +++ b/etl/hubspot/abri_flow_triggers.py @@ -0,0 +1,29 @@ +"""Builds the one-per-scrape SQS message for the Abri OpenHousing lambda. + +The scraper decides which flows fire (the differ predicates live in one +tested place); the lambda is a dumb dispatcher. When any predicate fires, +one message carries every fired flow plus the deal fields they need, so +flows for the same deal cannot interleave across concurrent lambda +invocations. + +Abandonment (no-access) detection stays unwired until the CancelJob flow +exists — check_for_abri_deal_abandonment is deliberately not called here. +""" + +from typing import Any, Dict, List, Optional + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer +from etl.hubspot.project_data import ProjectData +from etl.hubspot.utils import parse_hs_date + + +def build_abri_trigger_message( + hubspot_deal_id: str, + new_deal: Dict[str, str], + new_project: Optional[ProjectData], + new_listing: Optional[Dict[str, str]], + old_deal: HubspotDealData, +) -> Optional[Dict[str, Any]]: + """The Abri trigger message for this scrape, or None if no flow fired.""" + raise NotImplementedError diff --git a/etl/hubspot/tests/test_abri_flow_triggers.py b/etl/hubspot/tests/test_abri_flow_triggers.py new file mode 100644 index 000000000..63dbee4f9 --- /dev/null +++ b/etl/hubspot/tests/test_abri_flow_triggers.py @@ -0,0 +1,236 @@ +from datetime import datetime, timezone +from typing import Any, Dict +import uuid + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from etl.hubspot.abri_flow_triggers import build_abri_trigger_message + +BASE_TIME = datetime(2025, 12, 1, 12, 0, 0) + +DEAL_ID = "9876543210" + +LISTING = {"listing_id": "55", "owner_property_id": "1007165", "national_uprn": "1"} + + +def make_old_deal(**overrides: Any) -> HubspotDealData: + return HubspotDealData( + id=overrides.get("id", uuid.uuid4()), + deal_id=DEAL_ID, + created_at=BASE_TIME, + updated_at=BASE_TIME, + **{k: v for k, v in overrides.items() if k != "id"}, + ) + + +def make_new_deal(**overrides: str) -> Dict[str, str]: + return { + "hs_object_id": DEAL_ID, + "dealname": "49 Admers Crescent", + "project_code": "Abri Condition", + **overrides, + } + + +# ========================== +# SINGLE-FLOW FIRING +# ========================== + + +def test_a_first_confirmed_survey_date_builds_a_log_job_message() -> None: + # Arrange + old_deal = make_old_deal( + project_code="Abri Condition", confirmed_survey_date=None + ) + new_deal = make_new_deal( + confirmed_survey_date="2026-06-24", confirmed_survey_time="14:30" + ) + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message == { + "hubspot_deal_id": DEAL_ID, + "flows": ["log_job"], + "place_ref": "1007165", + "deal_name": "49 Admers Crescent", + "confirmed_survey_date": "2026-06-24", + "confirmed_survey_time": "14:30", + } + + +def test_a_changed_confirmed_survey_date_builds_an_amend_job_message() -> None: + # Arrange + old_deal = make_old_deal( + project_code="Abri Condition", + confirmed_survey_date=datetime(2026, 6, 18, tzinfo=timezone.utc), + ) + new_deal = make_new_deal( + confirmed_survey_date="2026-06-24", confirmed_survey_time="14:30" + ) + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is not None + assert message["flows"] == ["amend_job"] + assert message["confirmed_survey_date"] == "2026-06-24" + + +def test_a_first_expected_commencement_date_builds_a_tenant_sync_message() -> None: + # Arrange + old_deal = make_old_deal( + project_code="Abri Condition", expected_commencement_date=None + ) + new_deal = make_new_deal(expected_commencement_date="2026-07-01") + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is not None + assert message["flows"] == ["sync_tenant_data"] + assert message["place_ref"] == "1007165" + + +# ========================== +# ONE MESSAGE PER SCRAPE +# ========================== + + +def test_several_flows_firing_in_one_scrape_share_one_message() -> None: + # Arrange: date changed and commencement date first set together. + old_deal = make_old_deal( + project_code="Abri Condition", + confirmed_survey_date=datetime(2026, 6, 18, tzinfo=timezone.utc), + expected_commencement_date=None, + ) + new_deal = make_new_deal( + confirmed_survey_date="2026-06-24", + confirmed_survey_time="14:30", + expected_commencement_date="2026-07-01", + ) + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is not None + assert message["flows"] == ["amend_job", "sync_tenant_data"] + + +# ========================== +# NOTHING FIRES +# ========================== + + +def test_no_message_when_no_abri_trigger_fires() -> None: + # Arrange: something changed, but nothing Abri cares about. + old_deal = make_old_deal(project_code="Abri Condition", outcome=None) + new_deal = make_new_deal(outcome="left voicemail") + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is None + + +def test_no_message_for_a_non_abri_deal() -> None: + # Arrange: a survey date first set, but on another project's deal. + old_deal = make_old_deal(project_code="Other", confirmed_survey_date=None) + new_deal = make_new_deal( + project_code="Other", confirmed_survey_date="2026-06-24" + ) + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is None + + +def test_abandonment_alone_stays_unwired_and_builds_no_message() -> None: + # Arrange: the abandonment predicate would fire, but CancelJob does not + # exist yet, so no flow may be dispatched for it. + old_deal = make_old_deal( + project_code="Abri Condition", number_of_attempts="2", outcome=None + ) + new_deal = make_new_deal(number_of_attempts="3", outcome="no answer") + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=LISTING, + old_deal=old_deal, + ) + + # Assert + assert message is None + + +# ========================== +# PAYLOAD ROBUSTNESS +# ========================== + + +def test_a_missing_listing_still_builds_the_message_without_a_place_ref() -> None: + # Arrange: validation in the abri lambda is the visibility surface for + # a fired flow that lacks its place_ref; the scraper still sends. + old_deal = make_old_deal( + project_code="Abri Condition", confirmed_survey_date=None + ) + new_deal = make_new_deal(confirmed_survey_date="2026-06-24") + + # Act + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=new_deal, + new_project=None, + new_listing=None, + old_deal=old_deal, + ) + + # Assert + assert message is not None + assert message["place_ref"] is None + assert message["confirmed_survey_time"] is None From 28b4a8927f600e644d221d1abc4ac3a753dac89a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:13:48 +0000 Subject: [PATCH 15/34] =?UTF-8?q?A=20scrape=20that=20fires=20Abri=20trigge?= =?UTF-8?q?rs=20builds=20one=20message=20naming=20the=20fired=20flows=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/abri_flow_triggers.py | 32 ++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/etl/hubspot/abri_flow_triggers.py b/etl/hubspot/abri_flow_triggers.py index a379acd6f..6d72bbfcc 100644 --- a/etl/hubspot/abri_flow_triggers.py +++ b/etl/hubspot/abri_flow_triggers.py @@ -26,4 +26,34 @@ def build_abri_trigger_message( old_deal: HubspotDealData, ) -> Optional[Dict[str, Any]]: """The Abri trigger message for this scrape, or None if no flow fired.""" - raise NotImplementedError + flows: List[str] = [] + if HubspotDealDiffer.check_for_abri_job_amendment( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("amend_job") + if HubspotDealDiffer.check_for_abri_job_logging( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("log_job") + if HubspotDealDiffer.check_for_abri_tenant_data_fetch( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("sync_tenant_data") + + if not flows: + return None + + confirmed_survey_date = parse_hs_date(new_deal.get("confirmed_survey_date")) + listing = new_listing or {} + return { + "hubspot_deal_id": hubspot_deal_id, + "flows": flows, + "place_ref": listing.get("owner_property_id"), + "deal_name": new_deal.get("dealname"), + "confirmed_survey_date": ( + confirmed_survey_date.date().isoformat() + if confirmed_survey_date is not None + else None + ), + "confirmed_survey_time": new_deal.get("confirmed_survey_time"), + } From d70853bfd2fc84079276828e22ea3274839091d7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:16:13 +0000 Subject: [PATCH 16/34] =?UTF-8?q?The=20scraper=20sends=20fired=20Abri=20fl?= =?UTF-8?q?ows=20to=20the=20abri=20queue=20on=20each=20deal=20change=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- backend/app/config.py | 1 + etl/hubspot/scripts/scraper/main.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/backend/app/config.py b/backend/app/config.py index 8939e6ff8..df6a32d1d 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -40,6 +40,7 @@ class Settings(BaseSettings): CATEGORISATION_SQS_URL: str = "changeme" PASHUB_TO_ARA_SQS_URL: str = "changeme" MAGICPLAN_SQS_URL: str = "changeme" + ABRI_SQS_URL: str = "changeme" POSTCODE_SPLITTER_SQS_URL: str = "changeme" COMBINER_SQS_URL: str = "changeme" LANDLORD_OVERRIDES_SQS_URL: str = "changeme" diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 589e526b8..56ac8b9a5 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -6,6 +6,7 @@ from backend.app.config import get_settings from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.project_data import ProjectData +from etl.hubspot.abri_flow_triggers import build_abri_trigger_message from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, @@ -57,6 +58,8 @@ def handler(body: dict[str, Any], context: Any) -> None: # ============================== # Orchestration of other lambdas # ============================== + # No Abri triggers here: Abri deals are always created bare and their + # dates are set later by ops, so a first scrape can never fire one. if hubspot_deal["pashub_link"]: logger.info( f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" @@ -125,6 +128,26 @@ def handler(body: dict[str, Any], context: Any) -> None: sqs_client, hubspot_deal, listing, hubspot_deal_id ) + abri_message = build_abri_trigger_message( + hubspot_deal_id=hubspot_deal_id, + new_deal=hubspot_deal, + new_project=project, + new_listing=listing, + old_deal=db_deal, + ) + if abri_message: + logger.info( + f"Triggering Abri flows {abri_message['flows']} for HubSpot " + f"deal ID {hubspot_deal_id}" + ) + response = sqs_client.send_message( + QueueUrl=get_settings().ABRI_SQS_URL, + MessageBody=json.dumps(abri_message), + ) + logger.info( + f"Sent message to Abri queue. MessageId: {response['MessageId']}" + ) + print("done") From 8847d06d68863082f25a6d90f58d0f1267ed6fab Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:19:01 +0000 Subject: [PATCH 17/34] =?UTF-8?q?A=20first=20confirmed=20survey=20date=20o?= =?UTF-8?q?n=20an=20Abri=20deal=20reaches=20the=20abri=20queue=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/tests/test_scraper_handler.py | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index ec0a90eae..33452bccf 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -12,6 +12,7 @@ DEAL_ID = "999" PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" PASHUB_QUEUE_URL = "https://sqs.test/pashub" +ABRI_QUEUE_URL = "https://sqs.test/abri" PROJECT = {"project_id": "proj-1", "name": "Project One"} @@ -59,6 +60,7 @@ def run_handler( mock_boto3.client.return_value = mock_sqs mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = PASHUB_QUEUE_URL + mock_settings.return_value.ABRI_SQS_URL = ABRI_QUEUE_URL handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") @@ -229,6 +231,44 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: mock_sqs.send_message.assert_not_called() +# ========================================== +# EXISTING DEAL PATH - Abri trigger +# ========================================== + + +def test_existing_abri_deal_first_survey_date__sends_abri_sqs() -> None: + # Arrange + db_deal = make_db_deal( + project_code="Abri Condition", confirmed_survey_date=None + ) + hubspot_deal = make_hubspot_deal( + project_code="Abri Condition", + confirmed_survey_date="2026-06-24", + confirmed_survey_time="14:30", + ) + listing = {"owner_property_id": "1007165", "national_uprn": UPRN} + + # Act + mock_sqs, _ = run_handler( + hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing + ) + + # Assert + mock_sqs.send_message.assert_called_once_with( + QueueUrl=ABRI_QUEUE_URL, + MessageBody=json.dumps( + { + "hubspot_deal_id": DEAL_ID, + "flows": ["log_job"], + "place_ref": "1007165", + "deal_name": DEAL_NAME, + "confirmed_survey_date": "2026-06-24", + "confirmed_survey_time": "14:30", + } + ), + ) + + # ==================================== # PROJECT upsert # ==================================== From 58199a1c038c3838fb181583272044293c090e80 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:21:39 +0000 Subject: [PATCH 18/34] =?UTF-8?q?The=20abri=20lambda=20dispatches=20queue?= =?UTF-8?q?=20messages=20through=20the=20Abri=20orchestrator=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- applications/abri/handler.py | 57 ++++++++++++++++++++++ applications/abri/handler/Dockerfile | 18 +++++++ applications/abri/handler/requirements.txt | 8 +++ 3 files changed, 83 insertions(+) create mode 100644 applications/abri/handler.py create mode 100644 applications/abri/handler/Dockerfile create mode 100644 applications/abri/handler/requirements.txt diff --git a/applications/abri/handler.py b/applications/abri/handler.py new file mode 100644 index 000000000..bb6d6fe70 --- /dev/null +++ b/applications/abri/handler.py @@ -0,0 +1,57 @@ +"""Lambda entry point for the Abri OpenHousing flows. + +Consumes the abri SQS queue (one trigger message per scrape, sent by the +HubSpot scraper) and dispatches the fired flows through the AbriOrchestrator. +Env-to-clients glue only; behaviour lives behind the dispatch seam. +""" + +import os +from typing import Any, Dict + +from hubspot.client import Client # type: ignore[reportMissingTypeStubs] +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from applications.abri.abri_trigger_request import AbriTriggerRequest +from applications.abri.dispatch import dispatch_abri_flows +from domain.tasks.tasks import Source +from infrastructure.abri.abri_client import AbriClient +from infrastructure.abri.config import AbriConfig +from infrastructure.hubspot.deal_contacts_client import HubspotDealContactsClient +from infrastructure.hubspot.deal_properties_client import HubspotDealPropertiesClient +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine +from orchestration.abri_orchestrator import AbriOrchestrator +from repositories.hubspot_deals.deal_database_postgres_gateway import ( + DealDatabasePostgresGateway, +) +from utilities.aws_lambda.task_handler import task_handler +from utilities.logger import setup_logger + +logger = setup_logger() + + +@task_handler(task_source="abri", source=Source.HUBSPOT_DEAL) +def handler(body: dict[str, Any], context: Any) -> Dict[str, str]: + request = AbriTriggerRequest.model_validate(body) + + abri_client = AbriClient(config=AbriConfig.from_env(os.environ)) + sdk_client: Client = Client.create( # type: ignore[reportUnknownMemberType] + access_token=os.environ["HUBSPOT_API_KEY"] + ) + + # NullPool: a fresh engine per invocation, so pooling would only + # accumulate idle connections across warm Lambda containers. + engine = make_engine(PostgresConfig.from_env(dict(os.environ)), poolclass=NullPool) + with Session(engine) as session: + deal_database = DealDatabasePostgresGateway(session=session) + orchestrator = AbriOrchestrator( + abri_client=abri_client, + deal_contacts=HubspotDealContactsClient(sdk_client=sdk_client), + deal_properties=HubspotDealPropertiesClient(sdk_client=sdk_client), + deal_database=deal_database, + ) + summary = dispatch_abri_flows(request, orchestrator, deal_database) + + logger.info("Abri flows completed for deal %s: %s", request.hubspot_deal_id, summary) + return summary diff --git a/applications/abri/handler/Dockerfile b/applications/abri/handler/Dockerfile new file mode 100644 index 000000000..61aff08c3 --- /dev/null +++ b/applications/abri/handler/Dockerfile @@ -0,0 +1,18 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/abri/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utilities/ utilities/ +COPY utils/ utils/ +COPY backend/ backend/ +COPY applications/ applications/ +COPY domain/ domain/ +COPY datatypes/ datatypes/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY infrastructure/ infrastructure/ + +CMD ["applications.abri.handler.handler"] diff --git a/applications/abri/handler/requirements.txt b/applications/abri/handler/requirements.txt new file mode 100644 index 000000000..fafca4ae3 --- /dev/null +++ b/applications/abri/handler/requirements.txt @@ -0,0 +1,8 @@ +awslambdaric +requests +sqlalchemy==2.0.36 +sqlmodel +psycopg2-binary==2.9.10 +pydantic-settings==2.6.0 +boto3==1.35.44 +hubspot-api-client From 7b2fe806882fdfec43d9a3d7ebe2382a9b6ffbf3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:29:01 +0000 Subject: [PATCH 19/34] =?UTF-8?q?The=20abri=20lambda=20ships=20with=20its?= =?UTF-8?q?=20queue,=20DLQ=20and=20UAT-pointed=20staging=20config=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .github/workflows/_deploy_lambda.yml | 17 ++++ .github/workflows/deploy_terraform.yml | 45 +++++++++- deployment/terraform/lambda/abri/main.tf | 40 +++++++++ deployment/terraform/lambda/abri/outputs.tf | 9 ++ deployment/terraform/lambda/abri/provider.tf | 16 ++++ deployment/terraform/lambda/abri/variables.tf | 84 +++++++++++++++++++ .../terraform/lambda/hubspot_deal_etl/main.tf | 24 ++++++ deployment/terraform/shared/main.tf | 14 ++++ 8 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 deployment/terraform/lambda/abri/main.tf create mode 100644 deployment/terraform/lambda/abri/outputs.tf create mode 100644 deployment/terraform/lambda/abri/provider.tf create mode 100644 deployment/terraform/lambda/abri/variables.tf diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 9f6b07beb..e8e2de981 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -100,6 +100,15 @@ on: TF_VAR_open_epc_api_token: required: false + + TF_VAR_abri_relay_url: + required: false + TF_VAR_abri_relay_username: + required: false + TF_VAR_abri_relay_password: + required: false + TF_VAR_abri_relay_default_resource: + required: false jobs: deploy: runs-on: ubuntu-latest @@ -175,6 +184,10 @@ jobs: TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }} TF_VAR_openai_api_key: ${{ secrets.TF_VAR_openai_api_key }} TF_VAR_open_epc_api_token: ${{ secrets.TF_VAR_open_epc_api_token }} + TF_VAR_abri_relay_url: ${{ secrets.TF_VAR_abri_relay_url }} + TF_VAR_abri_relay_username: ${{ secrets.TF_VAR_abri_relay_username }} + TF_VAR_abri_relay_password: ${{ secrets.TF_VAR_abri_relay_password }} + TF_VAR_abri_relay_default_resource: ${{ secrets.TF_VAR_abri_relay_default_resource }} run: | ECR_REPO_URL_VAR="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then @@ -228,6 +241,10 @@ jobs: TF_VAR_magicplan_api_key: ${{ secrets.TF_VAR_magicplan_api_key }} TF_VAR_openai_api_key: ${{ secrets.TF_VAR_openai_api_key }} TF_VAR_open_epc_api_token: ${{ secrets.TF_VAR_open_epc_api_token }} + TF_VAR_abri_relay_url: ${{ secrets.TF_VAR_abri_relay_url }} + TF_VAR_abri_relay_username: ${{ secrets.TF_VAR_abri_relay_username }} + TF_VAR_abri_relay_password: ${{ secrets.TF_VAR_abri_relay_password }} + TF_VAR_abri_relay_default_resource: ${{ secrets.TF_VAR_abri_relay_default_resource }} run: | EXTRA_VARS="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 5e469dc66..468b0c6eb 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -700,6 +700,49 @@ jobs: TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }} TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }} + # ============================================================ + # Build Abri Lambda image + # ============================================================ + abri_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: abri-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: applications/abri/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Abri Lambda + # ============================================================ + abri_lambda: + needs: [abri_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: abri + lambda_path: deployment/terraform/lambda/abri + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: abri-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.abri_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_hubspot_api_key: ${{ secrets.HUBSPOT_API_KEY }} + # Staging points at Abri's UAT relay endpoint; production values are + # pending from Abri (prod apply is blocked until they exist). + TF_VAR_abri_relay_url: ${{ secrets.ABRI_RELAY_URL }} + TF_VAR_abri_relay_username: ${{ secrets.ABRI_RELAY_USERNAME }} + TF_VAR_abri_relay_password: ${{ secrets.ABRI_RELAY_PASSWORD }} + TF_VAR_abri_relay_default_resource: ${{ secrets.ABRI_RELAY_DEFAULT_RESOURCE }} + # ============================================================ # Build Audit Generator image # ============================================================ @@ -778,7 +821,7 @@ jobs: # Deploy Hubspot ETL Lambda # ============================================================ hubspot_etl_lambda: - needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda, magic_plan_lambda] + needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda, magic_plan_lambda, abri_lambda] uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: hubspot-etl-to-ara diff --git a/deployment/terraform/lambda/abri/main.tf b/deployment/terraform/lambda/abri/main.tf new file mode 100644 index 000000000..205e38eac --- /dev/null +++ b/deployment/terraform/lambda/abri/main.tf @@ -0,0 +1,40 @@ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "abri" + stage = var.stage + + image_uri = local.image_uri + + maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + + # Abri relay credentials are per-stage: the dev/staging stage points at + # Abri's UAT relay endpoint; production values are pending from Abri. + ABRI_RELAY_URL = var.abri_relay_url + ABRI_RELAY_USERNAME = var.abri_relay_username + ABRI_RELAY_PASSWORD = var.abri_relay_password + ABRI_RELAY_DEFAULT_RESOURCE = var.abri_relay_default_resource + + HUBSPOT_API_KEY = var.hubspot_api_key + + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_HOST = var.db_host + POSTGRES_DATABASE = var.db_name + POSTGRES_PORT = var.db_port + } +} diff --git a/deployment/terraform/lambda/abri/outputs.tf b/deployment/terraform/lambda/abri/outputs.tf new file mode 100644 index 000000000..aae33f3a5 --- /dev/null +++ b/deployment/terraform/lambda/abri/outputs.tf @@ -0,0 +1,9 @@ +output "abri_queue_url" { + value = module.lambda.queue_url + description = "URL of the Abri SQS queue" +} + +output "abri_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the Abri SQS queue" +} diff --git a/deployment/terraform/lambda/abri/provider.tf b/deployment/terraform/lambda/abri/provider.tf new file mode 100644 index 000000000..3ae50128a --- /dev/null +++ b/deployment/terraform/lambda/abri/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "abri-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/terraform/lambda/abri/variables.tf b/deployment/terraform/lambda/abri/variables.tf new file mode 100644 index 000000000..bef54cc41 --- /dev/null +++ b/deployment/terraform/lambda/abri/variables.tf @@ -0,0 +1,84 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda" + default = "abri" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null + description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} + +variable "abri_relay_url" { + type = string + sensitive = true +} + +variable "abri_relay_username" { + type = string + sensitive = true +} + +variable "abri_relay_password" { + type = string + sensitive = true +} + +variable "abri_relay_default_resource" { + type = string + sensitive = true +} + +variable "hubspot_api_key" { + type = string + sensitive = true +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} diff --git a/deployment/terraform/lambda/hubspot_deal_etl/main.tf b/deployment/terraform/lambda/hubspot_deal_etl/main.tf index ffb5f6f53..76459b9a4 100644 --- a/deployment/terraform/lambda/hubspot_deal_etl/main.tf +++ b/deployment/terraform/lambda/hubspot_deal_etl/main.tf @@ -25,6 +25,15 @@ data "terraform_remote_state" "magic_plan" { } } +data "terraform_remote_state" "abri" { + backend = "s3" + config = { + bucket = "abri-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + data "aws_secretsmanager_secret_version" "db_credentials" { secret_id = "${var.stage}/assessment_model/db_credentials" } @@ -59,6 +68,7 @@ module "hubspot_deal_etl" { PASHUB_TO_ARA_SQS_URL = data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_url MAGICPLAN_SQS_URL = data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_url + ABRI_SQS_URL = data.terraform_remote_state.abri.outputs.abri_queue_url } } @@ -100,4 +110,18 @@ module "hubspot_deal_etl_magicplan_sqs_policy" { resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_magicplan_sqs_send" { role = module.hubspot_deal_etl.role_name policy_arn = module.hubspot_deal_etl_magicplan_sqs_policy.policy_arn +} + +module "hubspot_deal_etl_abri_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "hubspot-deal-etl-abri-sqs-send-${var.stage}" + policy_description = "Allow HubSpot ETL Lambda to send messages to the Abri queue" + actions = ["sqs:SendMessage"] + resources = [data.terraform_remote_state.abri.outputs.abri_queue_arn] +} + +resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_abri_sqs_send" { + role = module.hubspot_deal_etl.role_name + policy_arn = module.hubspot_deal_etl_abri_sqs_policy.policy_arn } \ No newline at end of file diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index 3ff5bda4b..5cb1a93ac 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -894,3 +894,17 @@ output "modelling_e2e_ecr_url" { value = module.modelling_e2e_registry.repository_url } + +################################################ +# Abri OpenHousing – Lambda +################################################ +module "abri_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "abri-terraform-state" +} + +module "abri_registry" { + source = "../modules/container_registry" + name = "abri" + stage = var.stage +} From 45bb3a7e61466a1ef955cb191077c61c80ee69b1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 10:30:08 +0000 Subject: [PATCH 20/34] =?UTF-8?q?A=20manual=20smoke=20script=20proves=20lo?= =?UTF-8?q?g=20then=20amend=20end-to-end=20against=20Abri=20UAT=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- scripts/smoke_test_abri_log_amend.py | 145 +++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 scripts/smoke_test_abri_log_amend.py diff --git a/scripts/smoke_test_abri_log_amend.py b/scripts/smoke_test_abri_log_amend.py new file mode 100644 index 000000000..cc786f449 --- /dev/null +++ b/scripts/smoke_test_abri_log_amend.py @@ -0,0 +1,145 @@ +"""Manual smoke test for the Abri log -> amend wiring against UAT (issue #1480). + +Drives the real AbriOrchestrator log-job and amend-job flows end-to-end +against Abri's UAT relay endpoint: a job is logged for a test place +reference, its job_no is written back to a throwaway HubSpot deal, and the +appointment is then amended to the following day's other slot — proving the +full log -> write-back -> amend sequence converges. + +The deal-database gateway is in-memory: the write-back to the real deal-data +table belongs to the deployed lambda, and this script must not touch a +production database. + +Usage: + 1. Ensure these are set in the environment or backend/.env: + HUBSPOT_API_KEY, ABRI_RELAY_URL (the UAT endpoint), + ABRI_RELAY_USERNAME, ABRI_RELAY_PASSWORD, ABRI_RELAY_DEFAULT_RESOURCE. + 2. Create a throwaway test deal in the HubSpot UI and paste its id into + DEAL_ID below; paste Abri's agreed UAT test place reference into + PLACE_REF. + 3. Run: python scripts/smoke_test_abri_log_amend.py + 4. Check: the printed job_no matches the deal's client_booking_reference + in the HubSpot UI, and Abri can see the appointment on the amended + date/slot in their TM solution. + 5. Clean up: delete the test deal in the UI and ask Abri to cancel the + UAT job (there is no CancelJob route yet). +""" + +import os +from datetime import date, timedelta +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from dotenv import dotenv_values +from hubspot.client import Client # type: ignore[reportMissingTypeStubs] + +from domain.abri.models import AbriRequestRejected, PlaceRef +from infrastructure.abri.abri_client import AbriClient +from infrastructure.abri.config import AbriConfig +from infrastructure.hubspot.deal_contacts_client import HubspotDealContactsClient +from infrastructure.hubspot.deal_properties_client import HubspotDealPropertiesClient +from orchestration.abri_orchestrator import ( + AbriOrchestrator, + AppointmentChange, + ConfirmedSurveyBooking, +) + +DEAL_ID = "EDIT-ME" +PLACE_REF = PlaceRef("EDIT-ME") + +# Log for the first weekday at least a week out (a slot Abri UAT will take), +# then amend to the following day's opposite slot. +LOG_DATE = date.today() + timedelta(days=7) +LOG_TIME = "10:00" # AM slot +AMEND_DATE = LOG_DATE + timedelta(days=1) +AMEND_TIME = "14:30" # PM slot + + +def _env(name: str) -> str: + env_file = Path(__file__).parents[1] / "backend" / ".env" + value = os.getenv(name) or dotenv_values(env_file).get(name) + if not value: + raise SystemExit(f"Missing {name} (env var or backend/.env)") + return value + + +def _hubspot_sdk_client() -> Client: + return Client.create(access_token=_env("HUBSPOT_API_KEY")) # type: ignore[reportUnknownMemberType] + + +def _uat_abri_client() -> AbriClient: + return AbriClient( + config=AbriConfig( + endpoint_url=_env("ABRI_RELAY_URL"), + username=_env("ABRI_RELAY_USERNAME"), + password=_env("ABRI_RELAY_PASSWORD"), + default_resource=_env("ABRI_RELAY_DEFAULT_RESOURCE"), + ) + ) + + +class InMemoryDealDatabase: + """Stands in for the deal-data table; the real write-back is the lambda's.""" + + def __init__(self) -> None: + self.recorded: List[Tuple[str, str]] = [] + self._job_nos: Dict[str, str] = {} + + def record_job_no(self, deal_id: str, job_no: str) -> None: + self.recorded.append((deal_id, job_no)) + self._job_nos[deal_id] = job_no + + def job_no_for_deal(self, deal_id: str) -> Optional[str]: + return self._job_nos.get(deal_id) + + +def main() -> None: + if DEAL_ID == "EDIT-ME" or PLACE_REF == PlaceRef("EDIT-ME"): + raise SystemExit("Edit DEAL_ID and PLACE_REF before running") + + sdk_client = _hubspot_sdk_client() + deal_database = InMemoryDealDatabase() + orchestrator = AbriOrchestrator( + abri_client=_uat_abri_client(), + deal_contacts=HubspotDealContactsClient(sdk_client=sdk_client), + deal_properties=HubspotDealPropertiesClient(sdk_client=sdk_client), + deal_database=deal_database, + ) + + print(f"Logging job for place_ref {PLACE_REF} on {LOG_DATE} at {LOG_TIME}...") + log_result = orchestrator.log_job( + ConfirmedSurveyBooking( + deal_id=DEAL_ID, + place_ref=PLACE_REF, + deal_name=f"Domna UAT smoke test {date.today().isoformat()}", + confirmed_survey_date=LOG_DATE, + confirmed_survey_time=LOG_TIME, + ) + ) + if isinstance(log_result, AbriRequestRejected): + raise SystemExit( + f"LogJob rejected: {log_result.code}: {log_result.message}" + ) + print(f"Job logged: job_no={log_result.job_no} (written back to deal {DEAL_ID})") + + print(f"Amending appointment to {AMEND_DATE} at {AMEND_TIME}...") + amend_result = orchestrator.amend_job( + AppointmentChange( + deal_id=DEAL_ID, + confirmed_survey_date=AMEND_DATE, + confirmed_survey_time=AMEND_TIME, + ) + ) + if isinstance(amend_result, AbriRequestRejected): + raise SystemExit( + f"AmendJob rejected: {amend_result.code}: {amend_result.message}" + ) + print( + f"Appointment amended: job_no={amend_result.job_no} " + f"date={amend_result.appointment_date} slot={amend_result.appointment_time}" + ) + print("Smoke test complete. Remember to clean up (see module docstring).") + + +if __name__ == "__main__": + main() From eaedd9a63d4ae21e4071083b2bb9926123e1b625 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 11:44:51 +0000 Subject: [PATCH 21/34] =?UTF-8?q?Real=20Abri=20portal=20deals=20fire=20the?= =?UTF-8?q?=20job-logging=20trigger=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/tests/test_hubspot_deal_differ.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 5e9bcfdfa..12cb39425 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -453,6 +453,35 @@ def test_abri_job_logging__project_code_cased_differently__returns_true() -> Non assert result is True +def test_abri_job_logging__real_portal_project_code__returns_true() -> None: + # The code carried by every Abri stock-condition deal in the portal + # (verified 2026-07-07); these deals have no project association, so the + # project_code fallback is the branch that must fire in production. + deal_id = uuid.uuid4() + + # Arrange + old_deal = make_old_deal( + id=deal_id, + project_code="[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]", + confirmed_survey_date=None, + ) + new_deal = make_new_deal( + deal_id, + project_code="[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]", + confirmed_survey_date="2026-07-15", + ) + + # Act + result = HubspotDealDiffer.check_for_abri_job_logging( + new_deal=new_deal, + new_project=None, + old_deal=old_deal, + ) + + # Assert + assert result is True + + @pytest.mark.parametrize( "new_overrides", [ From 481c01098de037034cfb58e840db0d1a4f3e87ca Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 11:46:33 +0000 Subject: [PATCH 22/34] =?UTF-8?q?Real=20Abri=20portal=20deals=20fire=20the?= =?UTF-8?q?=20job-logging=20trigger=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/hubspot_deal_differ.py | 10 ++- etl/hubspot/tests/test_abri_flow_triggers.py | 20 +++-- etl/hubspot/tests/test_hubspot_deal_differ.py | 76 ++++++++++--------- etl/hubspot/tests/test_scraper_handler.py | 7 +- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 39a1956bb..c4395685b 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -21,8 +21,14 @@ class HubspotDealDiffer: "tenant refusal", "not viable", ] - ABRI_CONDITION_ASSOCIATED_PROJECT_ID = "123456" - ABRI_CONDITION_PROJECT_CODE = "Abri Condition" + # Verified against the portal (2026-07-07): Abri stock-condition deals + # carry no project association, so the project_code match is the branch + # that fires in production; the id below is the project object id + # embedded at the end of that code, kept for when associations appear. + ABRI_CONDITION_ASSOCIATED_PROJECT_ID = "1247536318670" + ABRI_CONDITION_PROJECT_CODE = ( + "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" + ) @staticmethod def check_for_db_update_trigger( diff --git a/etl/hubspot/tests/test_abri_flow_triggers.py b/etl/hubspot/tests/test_abri_flow_triggers.py index 63dbee4f9..9c7291562 100644 --- a/etl/hubspot/tests/test_abri_flow_triggers.py +++ b/etl/hubspot/tests/test_abri_flow_triggers.py @@ -7,6 +7,10 @@ from etl.hubspot.abri_flow_triggers import build_abri_trigger_message BASE_TIME = datetime(2025, 12, 1, 12, 0, 0) +ABRI_PROJECT_CODE = ( + "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" +) + DEAL_ID = "9876543210" LISTING = {"listing_id": "55", "owner_property_id": "1007165", "national_uprn": "1"} @@ -26,7 +30,7 @@ def make_new_deal(**overrides: str) -> Dict[str, str]: return { "hs_object_id": DEAL_ID, "dealname": "49 Admers Crescent", - "project_code": "Abri Condition", + "project_code": ABRI_PROJECT_CODE, **overrides, } @@ -39,7 +43,7 @@ def make_new_deal(**overrides: str) -> Dict[str, str]: def test_a_first_confirmed_survey_date_builds_a_log_job_message() -> None: # Arrange old_deal = make_old_deal( - project_code="Abri Condition", confirmed_survey_date=None + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None ) new_deal = make_new_deal( confirmed_survey_date="2026-06-24", confirmed_survey_time="14:30" @@ -68,7 +72,7 @@ def test_a_first_confirmed_survey_date_builds_a_log_job_message() -> None: def test_a_changed_confirmed_survey_date_builds_an_amend_job_message() -> None: # Arrange old_deal = make_old_deal( - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 6, 18, tzinfo=timezone.utc), ) new_deal = make_new_deal( @@ -93,7 +97,7 @@ def test_a_changed_confirmed_survey_date_builds_an_amend_job_message() -> None: def test_a_first_expected_commencement_date_builds_a_tenant_sync_message() -> None: # Arrange old_deal = make_old_deal( - project_code="Abri Condition", expected_commencement_date=None + project_code=ABRI_PROJECT_CODE, expected_commencement_date=None ) new_deal = make_new_deal(expected_commencement_date="2026-07-01") @@ -120,7 +124,7 @@ def test_a_first_expected_commencement_date_builds_a_tenant_sync_message() -> No def test_several_flows_firing_in_one_scrape_share_one_message() -> None: # Arrange: date changed and commencement date first set together. old_deal = make_old_deal( - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 6, 18, tzinfo=timezone.utc), expected_commencement_date=None, ) @@ -151,7 +155,7 @@ def test_several_flows_firing_in_one_scrape_share_one_message() -> None: def test_no_message_when_no_abri_trigger_fires() -> None: # Arrange: something changed, but nothing Abri cares about. - old_deal = make_old_deal(project_code="Abri Condition", outcome=None) + old_deal = make_old_deal(project_code=ABRI_PROJECT_CODE, outcome=None) new_deal = make_new_deal(outcome="left voicemail") # Act @@ -191,7 +195,7 @@ def test_abandonment_alone_stays_unwired_and_builds_no_message() -> None: # Arrange: the abandonment predicate would fire, but CancelJob does not # exist yet, so no flow may be dispatched for it. old_deal = make_old_deal( - project_code="Abri Condition", number_of_attempts="2", outcome=None + project_code=ABRI_PROJECT_CODE, number_of_attempts="2", outcome=None ) new_deal = make_new_deal(number_of_attempts="3", outcome="no answer") @@ -217,7 +221,7 @@ def test_a_missing_listing_still_builds_the_message_without_a_place_ref() -> Non # Arrange: validation in the abri lambda is the visibility surface for # a fired flow that lacks its place_ref; the scraper still sends. old_deal = make_old_deal( - project_code="Abri Condition", confirmed_survey_date=None + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None ) new_deal = make_new_deal(confirmed_survey_date="2026-06-24") diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 12cb39425..71d5892ea 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -11,6 +11,10 @@ from etl.hubspot.project_data import ProjectData BASE_TIME = datetime(2025, 12, 1, 12, 0, 0) +ABRI_PROJECT_CODE = ( + "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" +) + def make_old_deal(**overrides: Any) -> HubspotDealData: return HubspotDealData( @@ -355,12 +359,12 @@ def test_abri_job_logging__date_first_set_on_abri_deal__returns_true() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", ) @@ -381,12 +385,12 @@ def test_abri_job_logging__date_already_set__returns_false() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 7, 1, tzinfo=timezone.utc), ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", ) @@ -433,12 +437,12 @@ def test_abri_job_logging__project_code_cased_differently__returns_true() -> Non # Arrange old_deal = make_old_deal( id=deal_id, - project_code="ABRI CONDITION", + project_code=ABRI_PROJECT_CODE.upper(), confirmed_survey_date=None, ) new_deal = make_new_deal( deal_id, - project_code="ABRI CONDITION", + project_code=ABRI_PROJECT_CODE.upper(), confirmed_survey_date="2026-07-15", ) @@ -498,12 +502,12 @@ def test_abri_job_logging__no_parseable_new_date__returns_false( # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, **new_overrides, ) @@ -556,12 +560,12 @@ def test_abri_job_logging__non_abri_project_id_with_abri_code__returns_false() - # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", ) new_project = ProjectData(project_id="999999", name="Southern Retrofit") @@ -588,12 +592,12 @@ def test_abri_job_amendment__date_changed_on_abri_deal__returns_true() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 7, 1, tzinfo=timezone.utc), ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", ) @@ -640,13 +644,13 @@ def test_abri_job_amendment__date_and_time_unchanged__returns_false() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 7, 15, tzinfo=timezone.utc), confirmed_survey_time="09:30", ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", confirmed_survey_time="09:30", ) @@ -668,13 +672,13 @@ def test_abri_job_amendment__time_changed_on_abri_deal__returns_true() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=datetime(2026, 7, 15, tzinfo=timezone.utc), confirmed_survey_time="09:30", ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", confirmed_survey_time="14:00", ) @@ -696,12 +700,12 @@ def test_abri_job_amendment__date_first_set_on_abri_deal__returns_false() -> Non # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-07-15", ) @@ -729,12 +733,12 @@ def test_abri_tenant_data_fetch__commencement_date_first_set_on_abri_deal__retur # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date="2026-08-01", ) @@ -781,12 +785,12 @@ def test_abri_tenant_data_fetch__commencement_date_already_set__returns_false() # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date=datetime(2026, 7, 1, tzinfo=timezone.utc), ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date="2026-08-01", ) @@ -817,12 +821,12 @@ def test_abri_tenant_data_fetch__no_parseable_new_commencement_date__returns_fal # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, **new_overrides, ) @@ -875,12 +879,12 @@ def test_abri_tenant_data_fetch__non_abri_project_id_with_abri_code__returns_fal # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, expected_commencement_date="2026-08-01", ) new_project = ProjectData(project_id="999999", name="Southern Retrofit") @@ -909,12 +913,12 @@ def test_abri_deal_abandonment__third_attempt_with_negative_outcome__returns_tru # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, outcome=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, number_of_attempts="3", outcome="No Answer", ) @@ -974,12 +978,12 @@ def test_abri_deal_abandonment__fewer_than_three_parseable_attempts__returns_fal # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, outcome=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, outcome="No Answer", **new_overrides, ) @@ -1011,12 +1015,12 @@ def test_abri_deal_abandonment__outcome_not_negative__returns_false( # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, outcome=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, number_of_attempts="3", **new_overrides, ) @@ -1051,12 +1055,12 @@ def test_abri_deal_abandonment__each_negative_outcome__returns_true( # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, outcome=None, ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, number_of_attempts="3", outcome=outcome, ) @@ -1078,13 +1082,13 @@ def test_abri_deal_abandonment__deal_already_abandoned__returns_false() -> None: # Arrange old_deal = make_old_deal( id=deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, number_of_attempts="3", outcome="No Answer", ) new_deal = make_new_deal( deal_id, - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, number_of_attempts="4", outcome="No Answer", ) diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index 33452bccf..9963aa0e1 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -13,6 +13,9 @@ PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" PASHUB_QUEUE_URL = "https://sqs.test/pashub" ABRI_QUEUE_URL = "https://sqs.test/abri" +ABRI_PROJECT_CODE = ( + "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" +) PROJECT = {"project_id": "proj-1", "name": "Project One"} @@ -239,10 +242,10 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: def test_existing_abri_deal_first_survey_date__sends_abri_sqs() -> None: # Arrange db_deal = make_db_deal( - project_code="Abri Condition", confirmed_survey_date=None + project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None ) hubspot_deal = make_hubspot_deal( - project_code="Abri Condition", + project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-06-24", confirmed_survey_time="14:30", ) From 55baddfd9e69e9bfcf404aa6d72077bd751a645b Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 11:47:03 +0000 Subject: [PATCH 23/34] =?UTF-8?q?A=20local=20smoke=20drives=20HubSpot-to-w?= =?UTF-8?q?rite-back=20LogJob=20with=20only=20the=20Abri=20edge=20stubbed?= =?UTF-8?q?=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- scripts/smoke_test_abri_logjob_flow.py | 268 +++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 scripts/smoke_test_abri_logjob_flow.py diff --git a/scripts/smoke_test_abri_logjob_flow.py b/scripts/smoke_test_abri_logjob_flow.py new file mode 100644 index 000000000..0409ef3d1 --- /dev/null +++ b/scripts/smoke_test_abri_logjob_flow.py @@ -0,0 +1,268 @@ +"""Manual smoke test for the scraper->dispatch LogJob wiring (issue #1480). + +The closest run to end-to-end available before Abri send API credentials: +every seam is real except the Abri HTTP edge, which is stubbed to return a +canned LogJob success carrying a fake job_no. Concretely, the script + + 1. fetches the deal, listing and project from the real HubSpot portal + (exactly what the scraper lambda sees), + 2. reads the deal's row from the real (dev) deal database, + 3. runs the real differ predicates via build_abri_trigger_message, + 4. validates the real message contract (AbriTriggerRequest), + 5. runs the real dispatch through the real AbriOrchestrator, so the fake + job_no is written to the deal's client_booking_reference in HubSpot + first, then to the job_no column in the database, + 6. reads both back and prints them. + +NOTE: the LogJob trigger fires when the CONFIRMED SURVEY DATE is first set +on an Abri deal. (Setting the *expected commencement date* fires the +tenant-data flow instead — that flow is skipped here so the stub never +mints junk contacts; smoke_test_tenant_contacts.py covers it.) + +Usage: + 1. Ensure HUBSPOT_API_KEY and the postgres_* connection values are in + backend/.env (they are read tolerantly; the strict Settings .env + parsing is bypassed). + 2. Pick a test deal on the "Abri Condition" project that the scraper has + already ingested (its row must exist in hubspot_deal_data). Paste its + id into DEAL_ID below. + 3. In the HubSpot UI, set the deal's confirmed survey date (and + optionally a time). If the dev scraper lambda syncs the change into + the database before you run this, the differ will see no change — set + RESET_DB_ROW_FIRST = True to null the row's survey date and job_no so + the trigger fires deterministically. + 4. Run: python scripts/smoke_test_abri_logjob_flow.py + 5. Check the printed read-backs, and the deal in the HubSpot UI: the + fake job_no (SMOKE...) should sit in client_booking_reference. + 6. Run it again unchanged: the dispatch should print + "skipped: job SMOKE... already logged" — the idempotency guard. + 7. Clean up: set CLEANUP = True and re-run to clear the fake job_no from + both HubSpot and the database, then unset the deal's survey date. +""" + +import os +import xml.etree.ElementTree as ET +from datetime import datetime +from pathlib import Path +from typing import Any, List, Optional, cast + +from dotenv import dotenv_values + +DEAL_ID = "EDIT-ME" + +# Null the DB row's confirmed_survey_date + job_no before evaluating the +# differ, so the trigger fires even if the dev scraper already synced it. +RESET_DB_ROW_FIRST = False + +# Cleanup mode: clear the fake job_no from HubSpot and the database. +CLEANUP = False + +FAKE_JOB_NO = "SMOKE" + datetime.now().strftime("%d%H%M") + +JOB_NO_DEAL_PROPERTY = "client_booking_reference" + + +def _bootstrap_env() -> None: + """Export backend/.env tolerantly and bypass strict Settings parsing. + + backend/.env carries keys the strict Settings model rejects; exporting + them ourselves and pointing ENVIRONMENT away from "local" lets the + legacy HubspotClient (and PostgresConfig.from_env) run unchanged. + """ + env_file = Path(__file__).parents[1] / "backend" / ".env" + for key, value in dotenv_values(env_file).items(): + if value is not None: + os.environ.setdefault(key.upper(), value) + os.environ["ENVIRONMENT"] = "smoke-test" + + +class _CannedLogJobResponse: + content = ( + f'' + ).encode() + + def raise_for_status(self) -> None: + pass + + +class _StubAbriSession: + """Stands in for requests.Session; records envelopes, returns the canned + LogJob success. Anything but a logjob request is a wiring bug.""" + + def __init__(self) -> None: + self.sent_envelopes: List[bytes] = [] + + def post(self, url: str, data: bytes) -> _CannedLogJobResponse: + request_type = ( + ET.fromstring(data).find("Body/Request").get("request_type") # type: ignore[reportOptionalMemberAccess] + ) + if request_type != "logjob": + raise AssertionError( + f"stub received a {request_type!r} request; only logjob is smoke-tested" + ) + self.sent_envelopes.append(data) + return _CannedLogJobResponse() + + +def _print_sent_parameters(envelope: bytes) -> None: + parameters = { + parameter.get("attribute"): parameter.get("attribute_value") + for parameter in ET.fromstring(envelope).findall("Body/Request/Parameters") + } + print(" Envelope sent to (stubbed) Abri relay:") + for name, value in parameters.items(): + print(f" {name} = {value}") + + +def main() -> None: + if DEAL_ID == "EDIT-ME": + raise SystemExit("Edit DEAL_ID before running") + + _bootstrap_env() + + # Imports that read settings/env must come after the bootstrap. + from hubspot.client import Client # type: ignore[reportMissingTypeStubs] + from sqlalchemy.pool import NullPool + from sqlmodel import Session, select + + from applications.abri.abri_trigger_request import AbriTriggerRequest + from applications.abri.dispatch import dispatch_abri_flows + from backend.app.db.models.hubspot_deal_data import HubspotDealData + from etl.hubspot.abri_flow_triggers import build_abri_trigger_message + from etl.hubspot.hubspotClient import HubspotClient + from infrastructure.abri.abri_client import AbriClient + from infrastructure.abri.config import AbriConfig + from infrastructure.hubspot.deal_contacts_client import HubspotDealContactsClient + from infrastructure.hubspot.deal_properties_client import ( + HubspotDealPropertiesClient, + ) + from infrastructure.postgres.config import PostgresConfig + from infrastructure.postgres.engine import make_engine + from orchestration.abri_orchestrator import AbriOrchestrator + from repositories.hubspot_deals.deal_database_postgres_gateway import ( + DealDatabasePostgresGateway, + ) + + engine = make_engine(PostgresConfig.from_env(dict(os.environ)), poolclass=NullPool) + sdk_client: Client = Client.create( # type: ignore[reportUnknownMemberType] + access_token=os.environ["HUBSPOT_API_KEY"] + ) + deal_properties = HubspotDealPropertiesClient(sdk_client=sdk_client) + + with Session(engine) as session: + deal_row = session.exec( + select(HubspotDealData).where(HubspotDealData.deal_id == DEAL_ID) + ).first() + if deal_row is None: + raise SystemExit( + f"Deal {DEAL_ID} has no hubspot_deal_data row — let the scraper " + "ingest it first (the gateway write-back needs the row)." + ) + + if CLEANUP: + deal_properties.write_deal_property( + deal_id=DEAL_ID, property_name=JOB_NO_DEAL_PROPERTY, value="" + ) + deal_row.job_no = None + session.add(deal_row) + session.commit() + print( + f"Cleaned up: cleared {JOB_NO_DEAL_PROPERTY} in HubSpot and " + "job_no in the database. Unset the survey date in the UI too." + ) + return + + if RESET_DB_ROW_FIRST: + deal_row.confirmed_survey_date = None + deal_row.job_no = None + session.add(deal_row) + session.commit() + session.refresh(deal_row) + print("Reset DB row: confirmed_survey_date and job_no nulled.") + + print(f"Fetching deal {DEAL_ID} from HubSpot...") + hubspot_deal, _company, listing, project = HubspotClient( + ).get_deal_and_company_and_listing_and_project(DEAL_ID) + print( + f" dealname={hubspot_deal.get('dealname')!r} " + f"project_code={hubspot_deal.get('project_code')!r} " + f"confirmed_survey_date={hubspot_deal.get('confirmed_survey_date')!r} " + f"confirmed_survey_time={hubspot_deal.get('confirmed_survey_time')!r}" + ) + print( + f" DB row: confirmed_survey_date={deal_row.confirmed_survey_date} " + f"job_no={deal_row.job_no}" + ) + + message = build_abri_trigger_message( + hubspot_deal_id=DEAL_ID, + new_deal=hubspot_deal, + new_project=project, + new_listing=listing, + old_deal=deal_row, + ) + if message is None: + raise SystemExit( + "No Abri trigger fired. For LogJob, set the CONFIRMED SURVEY " + "DATE on the deal (expected commencement date fires the " + "tenant-data flow, not job logging). If the dev scraper " + "already synced your change, set RESET_DB_ROW_FIRST = True." + ) + print(f"Trigger message built: flows={message['flows']}") + + request = AbriTriggerRequest.model_validate(message) + if "log_job" not in request.flows: + raise SystemExit( + f"Fired flows {request.flows} do not include log_job — set the " + "confirmed survey date (not the expected commencement date)." + ) + if request.flows != ["log_job"]: + skipped = [flow for flow in request.flows if flow != "log_job"] + print(f" Restricting to log_job for this smoke; skipping {skipped}.") + request = request.model_copy(update={"flows": ["log_job"]}) + + stub_session = _StubAbriSession() + abri_client = AbriClient( + config=AbriConfig( + endpoint_url="https://stubbed.invalid/relay", + username="smoke-test", + password="", + default_resource="SMOKE", + ) + ) + abri_client._session = cast( # pyright: ignore[reportPrivateUsage] + Any, stub_session + ) + gateway = DealDatabasePostgresGateway(session=session) + orchestrator = AbriOrchestrator( + abri_client=abri_client, + deal_contacts=HubspotDealContactsClient(sdk_client=sdk_client), + deal_properties=deal_properties, + deal_database=gateway, + ) + + print("Dispatching...") + summary = dispatch_abri_flows(request, orchestrator, gateway) + for envelope in stub_session.sent_envelopes: + _print_sent_parameters(envelope) + print(f"Dispatch summary: {summary}") + + # Read both write-backs back through the real interfaces. + deal = sdk_client.crm.deals.basic_api.get_by_id( # type: ignore[reportUnknownMemberType] + DEAL_ID, properties=[JOB_NO_DEAL_PROPERTY] + ) + hubspot_value = cast( + Optional[str], + cast(Any, deal).properties.get(JOB_NO_DEAL_PROPERTY), + ) + print(f"HubSpot {JOB_NO_DEAL_PROPERTY} is now: {hubspot_value!r}") + print(f"Database job_no is now: {gateway.job_no_for_deal(DEAL_ID)!r}") + print( + "Re-run to see the idempotency guard skip the log; " + "set CLEANUP = True to undo." + ) + + +if __name__ == "__main__": + main() From 3b9b558628488b2d5012981cf711643d1d527dc9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 11:50:10 +0000 Subject: [PATCH 24/34] =?UTF-8?q?The=20job=20number=20is=20stored=20in=20t?= =?UTF-8?q?he=20client=5Fbooking=5Freference=20column=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../test_deal_database_postgres_gateway.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py index 89d4cde38..aaa96733b 100644 --- a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py +++ b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py @@ -2,7 +2,7 @@ from collections.abc import Iterator import pytest from sqlalchemy import Engine -from sqlmodel import Session +from sqlmodel import Session, text from backend.app.db.models.hubspot_deal_data import HubspotDealData from repositories.hubspot_deals.deal_database_postgres_gateway import ( @@ -48,6 +48,30 @@ def test_record_job_no_for_an_unknown_deal_raises(session: Session) -> None: gateway.record_job_no(deal_id="0000000000", job_no="AD0226519") +def test_the_job_no_is_stored_in_the_client_booking_reference_column( + session: Session, +) -> None: + # The physical column matches the HubSpot property the job number is + # stored under (client_booking_reference); only the domain calls it + # job_no. The deployed table is migrated by the frontend repo, so the + # column name here must match what that migration created. + # Arrange + _insert_deal(session, DEAL_ID) + gateway = DealDatabasePostgresGateway(session=session) + + # Act + gateway.record_job_no(deal_id=DEAL_ID, job_no="AD0226519") + + # Assert + stored = session.exec( # type: ignore[reportCallOverload] + text( + "select client_booking_reference from hubspot_deal_data " + "where deal_id = :deal_id" + ).bindparams(deal_id=DEAL_ID) + ).one() + assert stored[0] == "AD0226519" + + def test_job_no_for_deal_returns_the_recorded_job_no(session: Session) -> None: # Arrange _insert_deal(session, DEAL_ID) From 5286e2a2a20705003d70889bd58bc80254fe4d90 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 12:31:07 +0000 Subject: [PATCH 25/34] =?UTF-8?q?The=20job=20number=20is=20stored=20in=20t?= =?UTF-8?q?he=20client=5Fbooking=5Freference=20column=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- backend/app/db/models/hubspot_deal_data.py | 11 ++++++++--- .../test_deal_database_postgres_gateway.py | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index c221cbcdc..eab0e7b75 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -2,7 +2,7 @@ import uuid from sqlmodel import SQLModel, Field, Column, text from datetime import datetime from typing import Optional -from sqlalchemy import DateTime +from sqlalchemy import DateTime, String from sqlalchemy.sql import func @@ -71,8 +71,13 @@ class HubspotDealData(SQLModel, table=True): confirmed_survey_time: Optional[str] = Field(default=None) surveyed_date: Optional[datetime] = Field(default=None) design_type: Optional[str] = Field(default=None) - # OpenHousing job number; HubSpot property ID is client_booking_reference. - job_no: Optional[str] = Field(default=None) + # OpenHousing job number; stored under the same name as the HubSpot + # property that carries it (client_booking_reference) — the column is + # created by the frontend repo's drizzle migration. Only the domain + # calls it job_no. + job_no: Optional[str] = Field( + default=None, sa_column=Column("client_booking_reference", String) + ) survey_type: Optional[str] = Field(default=None) measures_for_pibi_ordered: Optional[str] = Field(default=None) diff --git a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py index aaa96733b..5e6a768ed 100644 --- a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py +++ b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py @@ -63,11 +63,12 @@ def test_the_job_no_is_stored_in_the_client_booking_reference_column( gateway.record_job_no(deal_id=DEAL_ID, job_no="AD0226519") # Assert - stored = session.exec( # type: ignore[reportCallOverload] + stored = session.connection().execute( text( "select client_booking_reference from hubspot_deal_data " "where deal_id = :deal_id" - ).bindparams(deal_id=DEAL_ID) + ), + {"deal_id": DEAL_ID}, ).one() assert stored[0] == "AD0226519" From a3985389ab5c764cfd43671ac689dc34298c6c26 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 12:43:40 +0000 Subject: [PATCH 26/34] =?UTF-8?q?The=20deal=20row=20names=20the=20job=20nu?= =?UTF-8?q?mber=20client=5Fbooking=5Freference,=20matching=20HubSpot=20and?= =?UTF-8?q?=20the=20schema=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- backend/app/db/models/hubspot_deal_data.py | 13 +++++------- etl/hubspot/hubspotDataTodB.py | 4 ++-- etl/hubspot/hubspot_deal_differ.py | 2 +- etl/hubspot/tests/test_hubspot_data_to_db.py | 12 ++++++----- etl/hubspot/tests/test_hubspot_deal_differ.py | 4 ++-- .../deal_database_postgres_gateway.py | 4 ++-- scripts/smoke_test_abri_logjob_flow.py | 15 +++++++------- .../test_deal_database_postgres_gateway.py | 20 +++++++++++-------- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index eab0e7b75..88bb5a69f 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -2,7 +2,7 @@ import uuid from sqlmodel import SQLModel, Field, Column, text from datetime import datetime from typing import Optional -from sqlalchemy import DateTime, String +from sqlalchemy import DateTime from sqlalchemy.sql import func @@ -71,13 +71,10 @@ class HubspotDealData(SQLModel, table=True): confirmed_survey_time: Optional[str] = Field(default=None) surveyed_date: Optional[datetime] = Field(default=None) design_type: Optional[str] = Field(default=None) - # OpenHousing job number; stored under the same name as the HubSpot - # property that carries it (client_booking_reference) — the column is - # created by the frontend repo's drizzle migration. Only the domain - # calls it job_no. - job_no: Optional[str] = Field( - default=None, sa_column=Column("client_booking_reference", String) - ) + # The OpenHousing job number, named after the HubSpot property that + # carries it (matching the column the frontend repo's drizzle migration + # created). Only the domain calls it job_no. + client_booking_reference: Optional[str] = Field(default=None) survey_type: Optional[str] = Field(default=None) measures_for_pibi_ordered: Optional[str] = Field(default=None) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 49719da3b..54c174c3f 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -274,7 +274,7 @@ class HubspotDataToDb: deal_data.get("confirmed_survey_date") ), "confirmed_survey_time": deal_data.get("confirmed_survey_time"), - "job_no": deal_data.get("client_booking_reference"), + "client_booking_reference": deal_data.get("client_booking_reference"), "surveyed_date": parse_hs_date(deal_data.get("surveyed_date")), "design_type": deal_data.get("design_type"), "survey_type": deal_data.get("survey_type"), @@ -383,7 +383,7 @@ class HubspotDataToDb: surveyor=deal_data.get("surveyor"), confirmed_survey_date=parse_hs_date(deal_data.get("confirmed_survey_date")), confirmed_survey_time=deal_data.get("confirmed_survey_time"), - job_no=deal_data.get("client_booking_reference"), + client_booking_reference=deal_data.get("client_booking_reference"), surveyed_date=parse_hs_date(deal_data.get("surveyed_date")), design_type=deal_data.get("design_type"), survey_type=deal_data.get("survey_type"), diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index c4395685b..6717be0e1 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -103,7 +103,7 @@ class HubspotDealDiffer: "design_type": "design_type", "surveyor": "surveyor", "confirmed_survey_time": "confirmed_survey_time", - "client_booking_reference": "job_no", + "client_booking_reference": "client_booking_reference", "survey_type": "survey_type", "measures_for_pibi_ordered": "measures_for_pibi_ordered", "property_halted_reason": "property_halted_reason", diff --git a/etl/hubspot/tests/test_hubspot_data_to_db.py b/etl/hubspot/tests/test_hubspot_data_to_db.py index 60c937460..8499ccd08 100644 --- a/etl/hubspot/tests/test_hubspot_data_to_db.py +++ b/etl/hubspot/tests/test_hubspot_data_to_db.py @@ -88,7 +88,7 @@ def test_update_existing_deal__no_project__clears_project_id() -> None: def test_update_existing_deal__client_booking_reference_maps_to_job_no() -> None: - existing = HubspotDealData(deal_id="MOCK_DEAL_ID", job_no=None) + existing = HubspotDealData(deal_id="MOCK_DEAL_ID", client_booking_reference=None) deal_data = {"client_booking_reference": "AD0226519"} _make_instance()._update_existing_deal( @@ -98,11 +98,13 @@ def test_update_existing_deal__client_booking_reference_maps_to_job_no() -> None company=None, ) - assert existing.job_no == "AD0226519" + assert existing.client_booking_reference == "AD0226519" def test_update_existing_deal__client_booking_reference_cleared__nulls_job_no() -> None: - existing = HubspotDealData(deal_id="MOCK_DEAL_ID", job_no="AD0226519") + existing = HubspotDealData( + deal_id="MOCK_DEAL_ID", client_booking_reference="AD0226519" + ) deal_data = {} _make_instance()._update_existing_deal( @@ -112,7 +114,7 @@ def test_update_existing_deal__client_booking_reference_cleared__nulls_job_no() company=None, ) - assert existing.job_no is None + assert existing.client_booking_reference is None def test_build_new_deal__client_booking_reference_maps_to_job_no() -> None: @@ -124,4 +126,4 @@ def test_build_new_deal__client_booking_reference_maps_to_job_no() -> None: project=None, ) - assert new_deal.job_no == "AD0226519" + assert new_deal.client_booking_reference == "AD0226519" diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 71d5892ea..2bd15996f 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -1267,7 +1267,7 @@ def test_db_update_trigger__client_booking_reference_changed__returns_true() -> # Arrange old_deal = make_old_deal( id=deal_id, - job_no=None, + client_booking_reference=None, ) new_deal = make_new_deal( deal_id, @@ -1293,7 +1293,7 @@ def test_db_update_trigger__client_booking_reference_unchanged__returns_false() # Arrange old_deal = make_old_deal( id=deal_id, - job_no="AD0226519", + client_booking_reference="AD0226519", ) new_deal = make_new_deal( deal_id, diff --git a/repositories/hubspot_deals/deal_database_postgres_gateway.py b/repositories/hubspot_deals/deal_database_postgres_gateway.py index bfd917781..5bc960fc3 100644 --- a/repositories/hubspot_deals/deal_database_postgres_gateway.py +++ b/repositories/hubspot_deals/deal_database_postgres_gateway.py @@ -21,13 +21,13 @@ class DealDatabasePostgresGateway: deal = self._find_deal(deal_id) if deal is None: raise ValueError(f"HubSpot deal {deal_id} not found") - deal.job_no = job_no + deal.client_booking_reference = job_no self._session.add(deal) self._session.commit() def job_no_for_deal(self, deal_id: str) -> Optional[str]: deal = self._find_deal(deal_id) - return deal.job_no if deal is not None else None + return deal.client_booking_reference if deal is not None else None def _find_deal(self, deal_id: str) -> Optional[HubspotDealData]: statement = select(HubspotDealData).where(HubspotDealData.deal_id == deal_id) diff --git a/scripts/smoke_test_abri_logjob_flow.py b/scripts/smoke_test_abri_logjob_flow.py index 0409ef3d1..f03f4004a 100644 --- a/scripts/smoke_test_abri_logjob_flow.py +++ b/scripts/smoke_test_abri_logjob_flow.py @@ -10,8 +10,8 @@ canned LogJob success carrying a fake job_no. Concretely, the script 3. runs the real differ predicates via build_abri_trigger_message, 4. validates the real message contract (AbriTriggerRequest), 5. runs the real dispatch through the real AbriOrchestrator, so the fake - job_no is written to the deal's client_booking_reference in HubSpot - first, then to the job_no column in the database, + job_no is written to the deal's client_booking_reference property in + HubSpot first, then to the same-named column in the database, 6. reads both back and prints them. NOTE: the LogJob trigger fires when the CONFIRMED SURVEY DATE is first set @@ -164,7 +164,7 @@ def main() -> None: deal_properties.write_deal_property( deal_id=DEAL_ID, property_name=JOB_NO_DEAL_PROPERTY, value="" ) - deal_row.job_no = None + deal_row.client_booking_reference = None session.add(deal_row) session.commit() print( @@ -175,15 +175,16 @@ def main() -> None: if RESET_DB_ROW_FIRST: deal_row.confirmed_survey_date = None - deal_row.job_no = None + deal_row.client_booking_reference = None session.add(deal_row) session.commit() session.refresh(deal_row) print("Reset DB row: confirmed_survey_date and job_no nulled.") print(f"Fetching deal {DEAL_ID} from HubSpot...") - hubspot_deal, _company, listing, project = HubspotClient( - ).get_deal_and_company_and_listing_and_project(DEAL_ID) + hubspot_deal, _company, listing, project = ( + HubspotClient().get_deal_and_company_and_listing_and_project(DEAL_ID) + ) print( f" dealname={hubspot_deal.get('dealname')!r} " f"project_code={hubspot_deal.get('project_code')!r} " @@ -192,7 +193,7 @@ def main() -> None: ) print( f" DB row: confirmed_survey_date={deal_row.confirmed_survey_date} " - f"job_no={deal_row.job_no}" + f"job_no={deal_row.client_booking_reference}" ) message = build_abri_trigger_message( diff --git a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py index 5e6a768ed..62b833686 100644 --- a/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py +++ b/tests/repositories/hubspot_deals/test_deal_database_postgres_gateway.py @@ -36,7 +36,7 @@ def test_record_job_no_persists_the_job_no_on_the_deal_row(session: Session) -> # Assert session.refresh(deal) - assert deal.job_no == "AD0226519" + assert deal.client_booking_reference == "AD0226519" def test_record_job_no_for_an_unknown_deal_raises(session: Session) -> None: @@ -63,13 +63,17 @@ def test_the_job_no_is_stored_in_the_client_booking_reference_column( gateway.record_job_no(deal_id=DEAL_ID, job_no="AD0226519") # Assert - stored = session.connection().execute( - text( - "select client_booking_reference from hubspot_deal_data " - "where deal_id = :deal_id" - ), - {"deal_id": DEAL_ID}, - ).one() + stored = ( + session.connection() + .execute( + text( + "select client_booking_reference from hubspot_deal_data " + "where deal_id = :deal_id" + ), + {"deal_id": DEAL_ID}, + ) + .one() + ) assert stored[0] == "AD0226519" From c016c8b754d5c334b13eb8200563e0c6e4ff4e75 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 12:58:48 +0000 Subject: [PATCH 27/34] =?UTF-8?q?Extract=20=5Ftrigger=5Fabri=5Flambda=20fo?= =?UTF-8?q?r=20consistency=20with=20the=20other=20scraper=20triggers=20?= =?UTF-8?q?=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/scripts/scraper/main.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 56ac8b9a5..89c49b867 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -140,13 +140,7 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Triggering Abri flows {abri_message['flows']} for HubSpot " f"deal ID {hubspot_deal_id}" ) - response = sqs_client.send_message( - QueueUrl=get_settings().ABRI_SQS_URL, - MessageBody=json.dumps(abri_message), - ) - logger.info( - f"Sent message to Abri queue. MessageId: {response['MessageId']}" - ) + _trigger_abri_lambda(sqs_client, abri_message) print("done") @@ -169,6 +163,14 @@ def _trigger_magicplan_fetcher( logger.info(f"Sent message to MagicPlan queue. MessageId: {response['MessageId']}") +def _trigger_abri_lambda(sqs_client: Any, abri_message: Dict[str, Any]) -> None: + response = sqs_client.send_message( + QueueUrl=get_settings().ABRI_SQS_URL, + MessageBody=json.dumps(abri_message), + ) + logger.info(f"Sent message to Abri queue. MessageId: {response['MessageId']}") + + def _trigger_pashub_fetcher( sqs_client: Any, deal_id: str, hubspot_deal: Dict[str, str] ) -> None: From d9b441962ff7717f22dd9d18e0eb0693e94ba9b1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 13:02:25 +0000 Subject: [PATCH 28/34] =?UTF-8?q?Name=20the=20Abri=20trigger=20builder=20f?= =?UTF-8?q?or=20its=20None-or-message=20contract=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/hubspot/abri_flow_triggers.py | 2 +- etl/hubspot/scripts/scraper/main.py | 8 +++++--- etl/hubspot/tests/test_abri_flow_triggers.py | 18 +++++++++--------- scripts/smoke_test_abri_logjob_flow.py | 6 +++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/etl/hubspot/abri_flow_triggers.py b/etl/hubspot/abri_flow_triggers.py index 6d72bbfcc..ccc1358b7 100644 --- a/etl/hubspot/abri_flow_triggers.py +++ b/etl/hubspot/abri_flow_triggers.py @@ -18,7 +18,7 @@ from etl.hubspot.project_data import ProjectData from etl.hubspot.utils import parse_hs_date -def build_abri_trigger_message( +def abri_trigger_message_for_scrape( hubspot_deal_id: str, new_deal: Dict[str, str], new_project: Optional[ProjectData], diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 89c49b867..78ce9cf65 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -6,7 +6,7 @@ from backend.app.config import get_settings from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.project_data import ProjectData -from etl.hubspot.abri_flow_triggers import build_abri_trigger_message +from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, @@ -128,14 +128,16 @@ def handler(body: dict[str, Any], context: Any) -> None: sqs_client, hubspot_deal, listing, hubspot_deal_id ) - abri_message = build_abri_trigger_message( + # None means no Abri flow fired this scrape; a dict is the message + # naming the flows that did. + abri_message = abri_trigger_message_for_scrape( hubspot_deal_id=hubspot_deal_id, new_deal=hubspot_deal, new_project=project, new_listing=listing, old_deal=db_deal, ) - if abri_message: + if abri_message is not None: logger.info( f"Triggering Abri flows {abri_message['flows']} for HubSpot " f"deal ID {hubspot_deal_id}" diff --git a/etl/hubspot/tests/test_abri_flow_triggers.py b/etl/hubspot/tests/test_abri_flow_triggers.py index 9c7291562..a586ab05c 100644 --- a/etl/hubspot/tests/test_abri_flow_triggers.py +++ b/etl/hubspot/tests/test_abri_flow_triggers.py @@ -3,7 +3,7 @@ from typing import Any, Dict import uuid from backend.app.db.models.hubspot_deal_data import HubspotDealData -from etl.hubspot.abri_flow_triggers import build_abri_trigger_message +from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape BASE_TIME = datetime(2025, 12, 1, 12, 0, 0) @@ -50,7 +50,7 @@ def test_a_first_confirmed_survey_date_builds_a_log_job_message() -> None: ) # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -80,7 +80,7 @@ def test_a_changed_confirmed_survey_date_builds_an_amend_job_message() -> None: ) # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -102,7 +102,7 @@ def test_a_first_expected_commencement_date_builds_a_tenant_sync_message() -> No new_deal = make_new_deal(expected_commencement_date="2026-07-01") # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -135,7 +135,7 @@ def test_several_flows_firing_in_one_scrape_share_one_message() -> None: ) # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -159,7 +159,7 @@ def test_no_message_when_no_abri_trigger_fires() -> None: new_deal = make_new_deal(outcome="left voicemail") # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -179,7 +179,7 @@ def test_no_message_for_a_non_abri_deal() -> None: ) # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -200,7 +200,7 @@ def test_abandonment_alone_stays_unwired_and_builds_no_message() -> None: new_deal = make_new_deal(number_of_attempts="3", outcome="no answer") # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -226,7 +226,7 @@ def test_a_missing_listing_still_builds_the_message_without_a_place_ref() -> Non new_deal = make_new_deal(confirmed_survey_date="2026-06-24") # Act - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, diff --git a/scripts/smoke_test_abri_logjob_flow.py b/scripts/smoke_test_abri_logjob_flow.py index f03f4004a..8f0de3246 100644 --- a/scripts/smoke_test_abri_logjob_flow.py +++ b/scripts/smoke_test_abri_logjob_flow.py @@ -7,7 +7,7 @@ canned LogJob success carrying a fake job_no. Concretely, the script 1. fetches the deal, listing and project from the real HubSpot portal (exactly what the scraper lambda sees), 2. reads the deal's row from the real (dev) deal database, - 3. runs the real differ predicates via build_abri_trigger_message, + 3. runs the real differ predicates via abri_trigger_message_for_scrape, 4. validates the real message contract (AbriTriggerRequest), 5. runs the real dispatch through the real AbriOrchestrator, so the fake job_no is written to the deal's client_booking_reference property in @@ -129,7 +129,7 @@ def main() -> None: from applications.abri.abri_trigger_request import AbriTriggerRequest from applications.abri.dispatch import dispatch_abri_flows from backend.app.db.models.hubspot_deal_data import HubspotDealData - from etl.hubspot.abri_flow_triggers import build_abri_trigger_message + from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape from etl.hubspot.hubspotClient import HubspotClient from infrastructure.abri.abri_client import AbriClient from infrastructure.abri.config import AbriConfig @@ -196,7 +196,7 @@ def main() -> None: f"job_no={deal_row.client_booking_reference}" ) - message = build_abri_trigger_message( + message = abri_trigger_message_for_scrape( hubspot_deal_id=DEAL_ID, new_deal=hubspot_deal, new_project=project, From 3934e2d3595f2740705b2c7830561755d281adaf Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 13:15:40 +0000 Subject: [PATCH 29/34] move trigger check to HubspotDealDiffer --- etl/hubspot/abri_flow_triggers.py | 59 -------------------- etl/hubspot/hubspot_deal_differ.py | 42 +++++++++++++- etl/hubspot/scripts/scraper/main.py | 18 +++--- etl/hubspot/tests/test_abri_flow_triggers.py | 34 +++++------ scripts/smoke_test_abri_logjob_flow.py | 8 ++- 5 files changed, 68 insertions(+), 93 deletions(-) delete mode 100644 etl/hubspot/abri_flow_triggers.py diff --git a/etl/hubspot/abri_flow_triggers.py b/etl/hubspot/abri_flow_triggers.py deleted file mode 100644 index ccc1358b7..000000000 --- a/etl/hubspot/abri_flow_triggers.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Builds the one-per-scrape SQS message for the Abri OpenHousing lambda. - -The scraper decides which flows fire (the differ predicates live in one -tested place); the lambda is a dumb dispatcher. When any predicate fires, -one message carries every fired flow plus the deal fields they need, so -flows for the same deal cannot interleave across concurrent lambda -invocations. - -Abandonment (no-access) detection stays unwired until the CancelJob flow -exists — check_for_abri_deal_abandonment is deliberately not called here. -""" - -from typing import Any, Dict, List, Optional - -from backend.app.db.models.hubspot_deal_data import HubspotDealData -from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer -from etl.hubspot.project_data import ProjectData -from etl.hubspot.utils import parse_hs_date - - -def abri_trigger_message_for_scrape( - hubspot_deal_id: str, - new_deal: Dict[str, str], - new_project: Optional[ProjectData], - new_listing: Optional[Dict[str, str]], - old_deal: HubspotDealData, -) -> Optional[Dict[str, Any]]: - """The Abri trigger message for this scrape, or None if no flow fired.""" - flows: List[str] = [] - if HubspotDealDiffer.check_for_abri_job_amendment( - new_deal=new_deal, new_project=new_project, old_deal=old_deal - ): - flows.append("amend_job") - if HubspotDealDiffer.check_for_abri_job_logging( - new_deal=new_deal, new_project=new_project, old_deal=old_deal - ): - flows.append("log_job") - if HubspotDealDiffer.check_for_abri_tenant_data_fetch( - new_deal=new_deal, new_project=new_project, old_deal=old_deal - ): - flows.append("sync_tenant_data") - - if not flows: - return None - - confirmed_survey_date = parse_hs_date(new_deal.get("confirmed_survey_date")) - listing = new_listing or {} - return { - "hubspot_deal_id": hubspot_deal_id, - "flows": flows, - "place_ref": listing.get("owner_property_id"), - "deal_name": new_deal.get("dealname"), - "confirmed_survey_date": ( - confirmed_survey_date.date().isoformat() - if confirmed_survey_date is not None - else None - ), - "confirmed_survey_time": new_deal.get("confirmed_survey_time"), - } diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 6717be0e1..e5b32ce29 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData from etl.hubspot.project_data import ProjectData @@ -189,6 +189,46 @@ class HubspotDealDiffer: return False + @staticmethod + def check_abri_triggers_and_construct_message( + hubspot_deal_id: str, + new_deal: Dict[str, str], + new_project: Optional[ProjectData], + new_listing: Optional[Dict[str, str]], + old_deal: HubspotDealData, + ) -> Optional[Dict[str, Any]]: + flows: List[str] = [] + if HubspotDealDiffer.check_for_abri_job_amendment( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("amend_job") + if HubspotDealDiffer.check_for_abri_job_logging( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("log_job") + if HubspotDealDiffer.check_for_abri_tenant_data_fetch( + new_deal=new_deal, new_project=new_project, old_deal=old_deal + ): + flows.append("sync_tenant_data") + + if not flows: + return None + + confirmed_survey_date = parse_hs_date(new_deal.get("confirmed_survey_date")) + listing = new_listing or {} + return { + "hubspot_deal_id": hubspot_deal_id, + "flows": flows, + "place_ref": listing.get("owner_property_id"), + "deal_name": new_deal.get("dealname"), + "confirmed_survey_date": ( + confirmed_survey_date.date().isoformat() + if confirmed_survey_date is not None + else None + ), + "confirmed_survey_time": new_deal.get("confirmed_survey_time"), + } + @staticmethod def check_for_abri_job_logging( new_deal: Dict[str, str], diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 78ce9cf65..c824852cd 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -6,7 +6,7 @@ from backend.app.config import get_settings from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.project_data import ProjectData -from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape +from etl.hubspot.abri_flow_triggers import check_for_abri_triggers_and_construct_message from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, @@ -128,14 +128,14 @@ def handler(body: dict[str, Any], context: Any) -> None: sqs_client, hubspot_deal, listing, hubspot_deal_id ) - # None means no Abri flow fired this scrape; a dict is the message - # naming the flows that did. - abri_message = abri_trigger_message_for_scrape( - hubspot_deal_id=hubspot_deal_id, - new_deal=hubspot_deal, - new_project=project, - new_listing=listing, - old_deal=db_deal, + abri_message: Optional[Dict[str, Any]] = ( + HubspotDealDiffer.check_abri_triggers_and_construct_message( + hubspot_deal_id=hubspot_deal_id, + new_deal=hubspot_deal, + new_project=project, + new_listing=listing, + old_deal=db_deal, + ) ) if abri_message is not None: logger.info( diff --git a/etl/hubspot/tests/test_abri_flow_triggers.py b/etl/hubspot/tests/test_abri_flow_triggers.py index a586ab05c..c6868b70b 100644 --- a/etl/hubspot/tests/test_abri_flow_triggers.py +++ b/etl/hubspot/tests/test_abri_flow_triggers.py @@ -3,13 +3,11 @@ from typing import Any, Dict import uuid from backend.app.db.models.hubspot_deal_data import HubspotDealData -from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape +from etl.hubspot.abri_flow_triggers import check_for_abri_triggers_and_construct_message BASE_TIME = datetime(2025, 12, 1, 12, 0, 0) -ABRI_PROJECT_CODE = ( - "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" -) +ABRI_PROJECT_CODE = "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" DEAL_ID = "9876543210" @@ -42,15 +40,13 @@ def make_new_deal(**overrides: str) -> Dict[str, str]: def test_a_first_confirmed_survey_date_builds_a_log_job_message() -> None: # Arrange - old_deal = make_old_deal( - project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None - ) + old_deal = make_old_deal(project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None) new_deal = make_new_deal( confirmed_survey_date="2026-06-24", confirmed_survey_time="14:30" ) # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -80,7 +76,7 @@ def test_a_changed_confirmed_survey_date_builds_an_amend_job_message() -> None: ) # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -102,7 +98,7 @@ def test_a_first_expected_commencement_date_builds_a_tenant_sync_message() -> No new_deal = make_new_deal(expected_commencement_date="2026-07-01") # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -135,7 +131,7 @@ def test_several_flows_firing_in_one_scrape_share_one_message() -> None: ) # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -159,7 +155,7 @@ def test_no_message_when_no_abri_trigger_fires() -> None: new_deal = make_new_deal(outcome="left voicemail") # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -174,12 +170,10 @@ def test_no_message_when_no_abri_trigger_fires() -> None: def test_no_message_for_a_non_abri_deal() -> None: # Arrange: a survey date first set, but on another project's deal. old_deal = make_old_deal(project_code="Other", confirmed_survey_date=None) - new_deal = make_new_deal( - project_code="Other", confirmed_survey_date="2026-06-24" - ) + new_deal = make_new_deal(project_code="Other", confirmed_survey_date="2026-06-24") # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -200,7 +194,7 @@ def test_abandonment_alone_stays_unwired_and_builds_no_message() -> None: new_deal = make_new_deal(number_of_attempts="3", outcome="no answer") # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, @@ -220,13 +214,11 @@ def test_abandonment_alone_stays_unwired_and_builds_no_message() -> None: def test_a_missing_listing_still_builds_the_message_without_a_place_ref() -> None: # Arrange: validation in the abri lambda is the visibility surface for # a fired flow that lacks its place_ref; the scraper still sends. - old_deal = make_old_deal( - project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None - ) + old_deal = make_old_deal(project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None) new_deal = make_new_deal(confirmed_survey_date="2026-06-24") # Act - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=new_deal, new_project=None, diff --git a/scripts/smoke_test_abri_logjob_flow.py b/scripts/smoke_test_abri_logjob_flow.py index 8f0de3246..36f0f40e8 100644 --- a/scripts/smoke_test_abri_logjob_flow.py +++ b/scripts/smoke_test_abri_logjob_flow.py @@ -7,7 +7,7 @@ canned LogJob success carrying a fake job_no. Concretely, the script 1. fetches the deal, listing and project from the real HubSpot portal (exactly what the scraper lambda sees), 2. reads the deal's row from the real (dev) deal database, - 3. runs the real differ predicates via abri_trigger_message_for_scrape, + 3. runs the real differ predicates via abri_message_if_triggered, 4. validates the real message contract (AbriTriggerRequest), 5. runs the real dispatch through the real AbriOrchestrator, so the fake job_no is written to the deal's client_booking_reference property in @@ -129,7 +129,9 @@ def main() -> None: from applications.abri.abri_trigger_request import AbriTriggerRequest from applications.abri.dispatch import dispatch_abri_flows from backend.app.db.models.hubspot_deal_data import HubspotDealData - from etl.hubspot.abri_flow_triggers import abri_trigger_message_for_scrape + from etl.hubspot.abri_flow_triggers import ( + check_for_abri_triggers_and_construct_message, + ) from etl.hubspot.hubspotClient import HubspotClient from infrastructure.abri.abri_client import AbriClient from infrastructure.abri.config import AbriConfig @@ -196,7 +198,7 @@ def main() -> None: f"job_no={deal_row.client_booking_reference}" ) - message = abri_trigger_message_for_scrape( + message = check_for_abri_triggers_and_construct_message( hubspot_deal_id=DEAL_ID, new_deal=hubspot_deal, new_project=project, From 252a146cc39fdc3f52825d386d531333cc460c0b Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 13:17:24 +0000 Subject: [PATCH 30/34] remove silly comment --- etl/hubspot/scripts/scraper/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index c824852cd..d51bd6f9c 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -58,8 +58,6 @@ def handler(body: dict[str, Any], context: Any) -> None: # ============================== # Orchestration of other lambdas # ============================== - # No Abri triggers here: Abri deals are always created bare and their - # dates are set later by ops, so a first scrape can never fire one. if hubspot_deal["pashub_link"]: logger.info( f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" From 7b6c7e1168750a1953e8ea4ed6e6e63f075e91d4 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 13:19:03 +0000 Subject: [PATCH 31/34] remove bad import --- etl/hubspot/scripts/scraper/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index d51bd6f9c..69480cfa4 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -6,7 +6,6 @@ from backend.app.config import get_settings from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.project_data import ProjectData -from etl.hubspot.abri_flow_triggers import check_for_abri_triggers_and_construct_message from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, From 04b32b1515ac785257f0c9991f01d9e39bb57808 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 13:21:01 +0000 Subject: [PATCH 32/34] minor refactor --- etl/hubspot/scripts/scraper/main.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 69480cfa4..cd78b70c5 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -110,17 +110,10 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) _trigger_pashub_fetcher(sqs_client, hubspot_deal_id, hubspot_deal) - else: - logger.info( - f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}" - ) if HubspotDealDiffer.check_for_magicplan_trigger( new_deal=hubspot_deal, old_deal=db_deal ): - logger.info( - f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}" - ) _trigger_magicplan_fetcher( sqs_client, hubspot_deal, listing, hubspot_deal_id ) @@ -139,7 +132,7 @@ def handler(body: dict[str, Any], context: Any) -> None: f"Triggering Abri flows {abri_message['flows']} for HubSpot " f"deal ID {hubspot_deal_id}" ) - _trigger_abri_lambda(sqs_client, abri_message) + _trigger_abri_api_lambda(sqs_client, abri_message) print("done") @@ -150,6 +143,8 @@ def _trigger_magicplan_fetcher( listing: Optional[dict[str, str]], hubspot_deal_id: str, ) -> None: + logger.info(f"Triggering MagicPlan fetcher for HubSpot deal ID {hubspot_deal_id}") + message_body = { "address": hubspot_deal.get("dealname"), "hubspot_deal_id": hubspot_deal_id, @@ -162,7 +157,7 @@ def _trigger_magicplan_fetcher( logger.info(f"Sent message to MagicPlan queue. MessageId: {response['MessageId']}") -def _trigger_abri_lambda(sqs_client: Any, abri_message: Dict[str, Any]) -> None: +def _trigger_abri_api_lambda(sqs_client: Any, abri_message: Dict[str, Any]) -> None: response = sqs_client.send_message( QueueUrl=get_settings().ABRI_SQS_URL, MessageBody=json.dumps(abri_message), From f5059c0589fdd47ad4ad29b0d908154ec7373073 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 14:04:34 +0000 Subject: [PATCH 33/34] tf comment --- .github/workflows/deploy_terraform.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 468b0c6eb..b9a4227ab 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -736,8 +736,7 @@ jobs: TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} TF_VAR_hubspot_api_key: ${{ secrets.HUBSPOT_API_KEY }} - # Staging points at Abri's UAT relay endpoint; production values are - # pending from Abri (prod apply is blocked until they exist). + # Awaiting API Key from Abri TF_VAR_abri_relay_url: ${{ secrets.ABRI_RELAY_URL }} TF_VAR_abri_relay_username: ${{ secrets.ABRI_RELAY_USERNAME }} TF_VAR_abri_relay_password: ${{ secrets.ABRI_RELAY_PASSWORD }} From c0e444fccbe597a6d19c2d401c8faacc26037452 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 7 Jul 2026 14:40:33 +0000 Subject: [PATCH 34/34] abri_api not abri --- .github/workflows/deploy_terraform.yml | 2 +- applications/abri/handler.py | 6 ++++-- .../lambda/{abri => abri_api}/main.tf | 2 +- .../lambda/{abri => abri_api}/outputs.tf | 4 ++-- .../lambda/{abri => abri_api}/provider.tf | 0 .../lambda/{abri => abri_api}/variables.tf | 2 +- .../terraform/lambda/hubspot_deal_etl/main.tf | 8 ++++---- deployment/terraform/shared/main.tf | 6 +++--- etl/hubspot/tests/test_scraper_handler.py | 18 ++++++++---------- .../utilities/aws_lambda/test_task_handler.py | 4 ++-- 10 files changed, 26 insertions(+), 26 deletions(-) rename deployment/terraform/lambda/{abri => abri_api}/main.tf (98%) rename deployment/terraform/lambda/{abri => abri_api}/outputs.tf (74%) rename deployment/terraform/lambda/{abri => abri_api}/provider.tf (100%) rename deployment/terraform/lambda/{abri => abri_api}/variables.tf (98%) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index a8b23bf9e..510ac147a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -718,7 +718,7 @@ jobs: # ============================================================ # Deploy Abri Lambda # ============================================================ - abri_lambda: + abri_api_lambda: needs: [abri_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: diff --git a/applications/abri/handler.py b/applications/abri/handler.py index bb6d6fe70..d9a80ae63 100644 --- a/applications/abri/handler.py +++ b/applications/abri/handler.py @@ -31,7 +31,7 @@ from utilities.logger import setup_logger logger = setup_logger() -@task_handler(task_source="abri", source=Source.HUBSPOT_DEAL) +@task_handler(task_source="abri_api", source=Source.HUBSPOT_DEAL) def handler(body: dict[str, Any], context: Any) -> Dict[str, str]: request = AbriTriggerRequest.model_validate(body) @@ -53,5 +53,7 @@ def handler(body: dict[str, Any], context: Any) -> Dict[str, str]: ) summary = dispatch_abri_flows(request, orchestrator, deal_database) - logger.info("Abri flows completed for deal %s: %s", request.hubspot_deal_id, summary) + logger.info( + "Abri flows completed for deal %s: %s", request.hubspot_deal_id, summary + ) return summary diff --git a/deployment/terraform/lambda/abri/main.tf b/deployment/terraform/lambda/abri_api/main.tf similarity index 98% rename from deployment/terraform/lambda/abri/main.tf rename to deployment/terraform/lambda/abri_api/main.tf index 205e38eac..a7bb1d6cf 100644 --- a/deployment/terraform/lambda/abri/main.tf +++ b/deployment/terraform/lambda/abri_api/main.tf @@ -9,7 +9,7 @@ locals { module "lambda" { source = "../../modules/lambda_with_sqs" - name = "abri" + name = "abri_api" stage = var.stage image_uri = local.image_uri diff --git a/deployment/terraform/lambda/abri/outputs.tf b/deployment/terraform/lambda/abri_api/outputs.tf similarity index 74% rename from deployment/terraform/lambda/abri/outputs.tf rename to deployment/terraform/lambda/abri_api/outputs.tf index aae33f3a5..7bbe29cd3 100644 --- a/deployment/terraform/lambda/abri/outputs.tf +++ b/deployment/terraform/lambda/abri_api/outputs.tf @@ -1,9 +1,9 @@ -output "abri_queue_url" { +output "abri_api_queue_url" { value = module.lambda.queue_url description = "URL of the Abri SQS queue" } -output "abri_queue_arn" { +output "abri_api_queue_arn" { value = module.lambda.queue_arn description = "ARN of the Abri SQS queue" } diff --git a/deployment/terraform/lambda/abri/provider.tf b/deployment/terraform/lambda/abri_api/provider.tf similarity index 100% rename from deployment/terraform/lambda/abri/provider.tf rename to deployment/terraform/lambda/abri_api/provider.tf diff --git a/deployment/terraform/lambda/abri/variables.tf b/deployment/terraform/lambda/abri_api/variables.tf similarity index 98% rename from deployment/terraform/lambda/abri/variables.tf rename to deployment/terraform/lambda/abri_api/variables.tf index bef54cc41..e431db62b 100644 --- a/deployment/terraform/lambda/abri/variables.tf +++ b/deployment/terraform/lambda/abri_api/variables.tf @@ -1,7 +1,7 @@ variable "lambda_name" { type = string description = "Logical name of the lambda" - default = "abri" + default = "abri_api" } variable "stage" { diff --git a/deployment/terraform/lambda/hubspot_deal_etl/main.tf b/deployment/terraform/lambda/hubspot_deal_etl/main.tf index 76459b9a4..0c06ea7c0 100644 --- a/deployment/terraform/lambda/hubspot_deal_etl/main.tf +++ b/deployment/terraform/lambda/hubspot_deal_etl/main.tf @@ -25,10 +25,10 @@ data "terraform_remote_state" "magic_plan" { } } -data "terraform_remote_state" "abri" { +data "terraform_remote_state" "abri_api" { backend = "s3" config = { - bucket = "abri-terraform-state" + bucket = "abri-api-terraform-state" key = "env:/${var.stage}/terraform.tfstate" region = "eu-west-2" } @@ -68,7 +68,7 @@ module "hubspot_deal_etl" { PASHUB_TO_ARA_SQS_URL = data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_url MAGICPLAN_SQS_URL = data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_url - ABRI_SQS_URL = data.terraform_remote_state.abri.outputs.abri_queue_url + ABRI_SQS_URL = data.terraform_remote_state.abri.outputs.abri_api_queue_url } } @@ -118,7 +118,7 @@ module "hubspot_deal_etl_abri_sqs_policy" { policy_name = "hubspot-deal-etl-abri-sqs-send-${var.stage}" policy_description = "Allow HubSpot ETL Lambda to send messages to the Abri queue" actions = ["sqs:SendMessage"] - resources = [data.terraform_remote_state.abri.outputs.abri_queue_arn] + resources = [data.terraform_remote_state.abri.outputs.abri_api_queue_arn] } resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_abri_sqs_send" { diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index 5cb1a93ac..b4cdafbcc 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -898,13 +898,13 @@ output "modelling_e2e_ecr_url" { ################################################ # Abri OpenHousing – Lambda ################################################ -module "abri_state_bucket" { +module "abri_api_state_bucket" { source = "../modules/tf_state_bucket" bucket_name = "abri-terraform-state" } -module "abri_registry" { +module "abri_api_registry" { source = "../modules/container_registry" - name = "abri" + name = "abri_api" stage = var.stage } diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index 9963aa0e1..c047153bb 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -12,10 +12,8 @@ DEAL_ID = "999" PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" PASHUB_QUEUE_URL = "https://sqs.test/pashub" -ABRI_QUEUE_URL = "https://sqs.test/abri" -ABRI_PROJECT_CODE = ( - "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" -) +ABRI_API_QUEUE_URL = "https://sqs.test/abri" +ABRI_PROJECT_CODE = "[Abri Stock Condition - Privately Funded - 062026 - 1247536318670]" PROJECT = {"project_id": "proj-1", "name": "Project One"} @@ -63,7 +61,7 @@ def run_handler( mock_boto3.client.return_value = mock_sqs mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL mock_settings.return_value.PASHUB_TO_ARA_SQS_URL = PASHUB_QUEUE_URL - mock_settings.return_value.ABRI_SQS_URL = ABRI_QUEUE_URL + mock_settings.return_value.ABRI_SQS_URL = ABRI_API_QUEUE_URL handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") @@ -131,7 +129,9 @@ def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None: listing = {"national_uprn": UPRN} # Act - mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) + mock_sqs, _ = run_handler( + hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing + ) # Assert mock_sqs.send_message.assert_called_once_with( @@ -241,9 +241,7 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: def test_existing_abri_deal_first_survey_date__sends_abri_sqs() -> None: # Arrange - db_deal = make_db_deal( - project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None - ) + db_deal = make_db_deal(project_code=ABRI_PROJECT_CODE, confirmed_survey_date=None) hubspot_deal = make_hubspot_deal( project_code=ABRI_PROJECT_CODE, confirmed_survey_date="2026-06-24", @@ -258,7 +256,7 @@ def test_existing_abri_deal_first_survey_date__sends_abri_sqs() -> None: # Assert mock_sqs.send_message.assert_called_once_with( - QueueUrl=ABRI_QUEUE_URL, + QueueUrl=ABRI_API_QUEUE_URL, MessageBody=json.dumps( { "hubspot_deal_id": DEAL_ID, diff --git a/tests/utilities/aws_lambda/test_task_handler.py b/tests/utilities/aws_lambda/test_task_handler.py index 1d9ad84cd..5ab28a2ff 100644 --- a/tests/utilities/aws_lambda/test_task_handler.py +++ b/tests/utilities/aws_lambda/test_task_handler.py @@ -110,7 +110,7 @@ def test_task_handler_reports_an_ordinarily_failing_record_for_redelivery( ) -> None: # Arrange @task_handler( - task_source="abri", + task_source="abri_api", source=Source.HUBSPOT_DEAL, orchestrator_cm=harness.factory, ) @@ -133,7 +133,7 @@ def test_task_handler_does_not_requeue_a_record_failing_non_retriably( ) -> None: # Arrange @task_handler( - task_source="abri", + task_source="abri_api", source=Source.HUBSPOT_DEAL, orchestrator_cm=harness.factory, )