Your document package is ready.
" + f"It contains {included} document(s) across " + f"{properties} {noun}.{skip_note}
" + '' + f'Download documents
' + f'This link expires in {minutes} '
+ "minutes. If the button doesn't work, paste this URL into your browser:"
+ f'
{safe_url}
Download documents
", + ) + + # Assert — a multipart/alternative carrying both the plain text and the HTML. + message = created[0].messages[0] + assert message.get_content_type() == "multipart/alternative" + plain_part = message.get_body(preferencelist=("plain",)) + html_part = message.get_body(preferencelist=("html",)) + assert plain_part is not None and "plain text fallback" in plain_part.get_content() + assert html_part is not None and "Download documents" in html_part.get_content() + + +def test_send_starts_tls_and_authenticates_before_sending() -> None: + # Arrange — SES SMTP rejects unauthenticated, cleartext senders, so the + # transport must STARTTLS and log in with the SMTP credentials. + created: list[_FakeSmtp] = [] + + def factory(host: str, port: int) -> _FakeSmtp: + smtp = _FakeSmtp(host, port) + created.append(smtp) + return smtp + + sender = SesSmtpEmailSender( + host="email-smtp.eu-west-2.amazonaws.com", + port=587, + username="AKIA_SMTP_USER", + password="smtp-password", + from_address="noreply@domna.homes", + smtp_factory=factory, + ) + + # Act + sender.send(to="user@example.com", subject="s", body="b") + + # Assert + assert created[0].started_tls is True + assert created[0].login_args == ("AKIA_SMTP_USER", "smtp-password") diff --git a/tests/infrastructure/test_s3_client.py b/tests/infrastructure/test_s3_client.py index bdac6be1e..13e6c4714 100644 --- a/tests/infrastructure/test_s3_client.py +++ b/tests/infrastructure/test_s3_client.py @@ -34,3 +34,33 @@ def test_get_object_returns_bytes_written_by_put_object(s3_client: S3Client) -> def test_bucket_property_exposes_configured_bucket(s3_client: S3Client) -> None: # act / assert assert s3_client.bucket == BUCKET + + +def test_generate_presigned_url_signs_a_get_for_the_object(s3_client: S3Client) -> None: + # arrange + s3_client.put_object("bulk-downloads/pkg.zip", b"zip-bytes") + + # act + url = s3_client.generate_presigned_url("bulk-downloads/pkg.zip", expires_in=3600) + + # assert — a signed GET URL for this bucket + key + assert BUCKET in url + assert "bulk-downloads/pkg.zip" in url + assert "Signature" in url + + +def test_upload_file_streams_a_local_file_to_s3( + s3_client: S3Client, tmp_path: object +) -> None: + # arrange — a file on local disk (stands in for the /tmp ZIP) + from pathlib import Path + + local = Path(str(tmp_path)) / "package.zip" + local.write_bytes(b"ZIP-CONTENT") + + # act + uri = s3_client.upload_file(str(local), "bulk-downloads/package.zip") + + # assert — the object landed with the file's bytes + assert uri == f"s3://{BUCKET}/bulk-downloads/package.zip" + assert s3_client.get_object("bulk-downloads/package.zip") == b"ZIP-CONTENT" diff --git a/tests/infrastructure/test_s3_document_downloader.py b/tests/infrastructure/test_s3_document_downloader.py new file mode 100644 index 000000000..58753d2b1 --- /dev/null +++ b/tests/infrastructure/test_s3_document_downloader.py @@ -0,0 +1,36 @@ +"""The multi-bucket document downloader (ADR-0060) — streams object bytes from +whichever source bucket an uploaded file lives in, to local disk.""" + +from collections.abc import Iterator +from pathlib import Path + +import pytest +from moto import mock_aws + +from infrastructure.s3.s3_document_downloader import S3DocumentDownloader +from tests.infrastructure import make_boto_client + + +@pytest.fixture +def downloader() -> Iterator[S3DocumentDownloader]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket="src-a") + boto_client.create_bucket(Bucket="src-b") + boto_client.put_object(Bucket="src-a", Key="surveys/one.pdf", Body=b"AAA") + boto_client.put_object(Bucket="src-b", Key="photos/two.zip", Body=b"BBB") + yield S3DocumentDownloader(boto_client) + + +def test_downloads_bytes_from_the_named_source_bucket( + downloader: S3DocumentDownloader, tmp_path: Path +) -> None: + # act — each object streams from its own bucket to a local file. + a = tmp_path / "a" + b = tmp_path / "b" + downloader.download("src-a", "surveys/one.pdf", str(a)) + downloader.download("src-b", "photos/two.zip", str(b)) + + # assert + assert a.read_bytes() == b"AAA" + assert b.read_bytes() == b"BBB" diff --git a/tests/orchestration/test_bulk_document_download_orchestrator.py b/tests/orchestration/test_bulk_document_download_orchestrator.py new file mode 100644 index 000000000..01cdf9634 --- /dev/null +++ b/tests/orchestration/test_bulk_document_download_orchestrator.py @@ -0,0 +1,286 @@ +"""The Bulk Document Download orchestrator (ADR-0060): gather a property set's +files, lay them out, zip + upload, presign, and email the requester.""" + +from __future__ import annotations + +import io +import zipfile +from collections.abc import Iterator +from datetime import datetime, timezone +from typing import Optional + +import pytest +from moto import mock_aws + +from domain.bulk_document_download.property_document import PropertyDocument +from domain.tasks.subtasks import SubTaskFailure +from infrastructure.postgres.uploaded_file_table import FileTypeEnum +from infrastructure.s3.s3_client import S3Client +from orchestration.bulk_document_download_orchestrator import ( + BulkDocumentDownloadOrchestrator, +) +from tests.infrastructure import make_boto_client + +DATA_BUCKET = "data-bucket" + + +class _FakeUploadedFiles: + def __init__(self, documents: list[PropertyDocument]) -> None: + self._documents = documents + + def by_landlord_property_ids( + self, landlord_property_ids: object + ) -> list[PropertyDocument]: + wanted = set(landlord_property_ids) # type: ignore[call-overload] + return [d for d in self._documents if d.landlord_property_id in wanted] + + +class _FakeAddresses: + def __init__(self, mapping: dict[str, str]) -> None: + self._mapping = mapping + + def address_for(self, landlord_property_id: str) -> str: + return self._mapping[landlord_property_id] + + +class _FakeDocuments: + def __init__(self, blobs: dict[tuple[str, str], bytes]) -> None: + self._blobs = blobs + + def download(self, bucket: str, key: str, dest_path: str) -> None: + # KeyError for an object we don't hold — stands in for a missing/ + # unreadable source object. + with open(dest_path, "wb") as handle: + handle.write(self._blobs[(bucket, key)]) + + +class _RecordingEmail: + def __init__(self) -> None: + self.sent: list[tuple[str, str, str, Optional[str]]] = [] + + def send( + self, *, to: str, subject: str, body: str, html_body: Optional[str] = None + ) -> None: + self.sent.append((to, subject, body, html_body)) + + +class _FailingEmail: + def send( + self, *, to: str, subject: str, body: str, html_body: Optional[str] = None + ) -> None: + raise RuntimeError("smtp unreachable") + + +def _document( + landlord_property_id: str, + s3_bucket: str, + s3_key: str, + file_type: Optional[FileTypeEnum] = FileTypeEnum.SITE_NOTE, +) -> PropertyDocument: + return PropertyDocument( + landlord_property_id=landlord_property_id, + document_type=file_type.value if file_type is not None else None, + s3_bucket=s3_bucket, + s3_key=s3_key, + uploaded_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + + +@pytest.fixture +def packages() -> Iterator[S3Client]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket=DATA_BUCKET) + yield S3Client(boto_client, DATA_BUCKET) + + +def test_builds_uploads_and_emails_a_download_package(packages: S3Client) -> None: + # Arrange — one property with one site note. + files = [_document("LP1", "src-bucket", "surveys/a.pdf")] + email = _RecordingEmail() + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles(files), + addresses=_FakeAddresses({"LP1": "12 Oak Street"}), + documents=_FakeDocuments({("src-bucket", "surveys/a.pdf"): b"PDF-BYTES"}), + packages=packages, + email=email, + url_ttl_seconds=3600, + ) + + # Act + result = orchestrator.run( + landlord_property_ids=["LP1"], + recipient_email="user@example.com", + package_name="portfolio-42", + ) + + # Assert — one file packaged, at the planned path with the source bytes. + assert result.included == 1 + archive = packages.get_object(result.package_s3_key) + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + assert zf.read("12 Oak Street (LP1)/site_note.pdf") == b"PDF-BYTES" + + # ...the presigned URL points at that object, and the requester is emailed it. + assert result.package_s3_key in result.presigned_url + assert len(email.sent) == 1 + to, subject, body, html_body = email.sent[0] + assert to == "user@example.com" + assert "1 documents" in subject + assert result.presigned_url in body # raw URL in the plain-text fallback + assert html_body is not None + assert "Download documents" in html_body # clean link, not the raw URL + + +def test_an_email_failure_does_not_lose_the_built_package(packages: S3Client) -> None: + # Arrange — the package builds and uploads fine, but the email transport is + # down. Best-effort delivery (ADR-0060): the URL is still returned (→ outputs). + files = [_document("LP1", "src-bucket", "surveys/a.pdf")] + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles(files), + addresses=_FakeAddresses({"LP1": "12 Oak Street"}), + documents=_FakeDocuments({("src-bucket", "surveys/a.pdf"): b"PDF-BYTES"}), + packages=packages, + email=_FailingEmail(), + url_ttl_seconds=3600, + ) + + # Act — must not raise despite the email failure. + result = orchestrator.run( + landlord_property_ids=["LP1"], + recipient_email="user@example.com", + package_name="email-down", + ) + + # Assert — the package was still uploaded and its URL returned. + assert result.included == 1 + assert result.package_s3_key in result.presigned_url + assert packages.get_object(result.package_s3_key) # uploaded despite email failure + + +def test_a_selection_with_no_documents_is_a_recorded_failure( + packages: S3Client, +) -> None: + # Arrange — the selection resolves to no packageable documents. + email = _RecordingEmail() + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles([]), + addresses=_FakeAddresses({}), + documents=_FakeDocuments({}), + packages=packages, + email=email, + url_ttl_seconds=3600, + ) + + # Act / Assert — a recorded SubTask failure, and nobody is emailed an empty + # package. + with pytest.raises(SubTaskFailure) as exc_info: + orchestrator.run( + landlord_property_ids=["LP1", "LP2"], + recipient_email="user@example.com", + package_name="empty", + ) + assert email.sent == [] + + # ...and it carries the stage counts so the failure is debuggable: the + # message (→ the worker's WARNING log) and details (→ sub_task.outputs). + failure = exc_info.value + assert "matched 0 documents" in str(failure) + assert failure.details is not None + assert failure.details["selected_properties"] == 2 + assert failure.details["matched_documents"] == 0 + + +def test_packages_the_good_documents_and_reports_the_skipped_ones( + packages: S3Client, +) -> None: + # Arrange — one packable site note and one row with no Document Type. + good = _document("LP1", "src-bucket", "surveys/a.pdf") + untyped = _document("LP1", "src-bucket", "mystery.bin", file_type=None) + email = _RecordingEmail() + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles([good, untyped]), + addresses=_FakeAddresses({"LP1": "12 Oak Street"}), + documents=_FakeDocuments({("src-bucket", "surveys/a.pdf"): b"PDF-BYTES"}), + packages=packages, + email=email, + url_ttl_seconds=3600, + ) + + # Act + result = orchestrator.run( + landlord_property_ids=["LP1"], + recipient_email="user@example.com", + package_name="mixed", + ) + + # Assert — the good file is packaged and delivered; the untyped one is + # reported, not fatal. + assert result.included == 1 + assert [s.s3_key for s in result.skipped] == ["mystery.bin"] + assert len(email.sent) == 1 + archive = packages.get_object(result.package_s3_key) + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + assert zf.namelist() == ["12 Oak Street (LP1)/site_note.pdf"] + + +def test_a_file_that_cannot_be_read_is_skipped_not_fatal(packages: S3Client) -> None: + # Arrange — two documents for one property; the photo pack's object is + # missing (download raises), the site note reads fine. Best-effort (ADR-0060): + # the run must still deliver the readable file and report the other. + good = _document("LP1", "src-bucket", "surveys/a.pdf") + missing = _document( + "LP1", "src-bucket", "surveys/gone.pdf", file_type=FileTypeEnum.PHOTO_PACK + ) + email = _RecordingEmail() + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles([good, missing]), + addresses=_FakeAddresses({"LP1": "12 Oak Street"}), + documents=_FakeDocuments({("src-bucket", "surveys/a.pdf"): b"PDF-BYTES"}), + packages=packages, + email=email, + url_ttl_seconds=3600, + ) + + # Act + result = orchestrator.run( + landlord_property_ids=["LP1"], + recipient_email="user@example.com", + package_name="partial", + ) + + # Assert — the readable file is delivered; the missing one is reported. + assert result.included == 1 + assert [(s.s3_key, s.reason) for s in result.skipped] == [ + ("surveys/gone.pdf", "unreadable") + ] + assert len(email.sent) == 1 + archive = packages.get_object(result.package_s3_key) + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + assert zf.namelist() == ["12 Oak Street (LP1)/site_note.pdf"] + + +def test_the_address_fallback_string_becomes_the_folder_name( + packages: S3Client, +) -> None: + # Arrange — a property whose address resolved to the fallback string. + doc = _document("LP9", "src-bucket", "surveys/a.pdf") + orchestrator = BulkDocumentDownloadOrchestrator( + uploaded_files=_FakeUploadedFiles([doc]), + addresses=_FakeAddresses({"LP9": "address unavailable"}), + documents=_FakeDocuments({("src-bucket", "surveys/a.pdf"): b"PDF-BYTES"}), + packages=packages, + email=_RecordingEmail(), + url_ttl_seconds=3600, + ) + + # Act + result = orchestrator.run( + landlord_property_ids=["LP9"], + recipient_email="user@example.com", + package_name="fallback", + ) + + # Assert — the fallback flows through to a usable folder name. + archive = packages.get_object(result.package_s3_key) + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + assert zf.namelist() == ["address unavailable (LP9)/site_note.pdf"] diff --git a/tests/repositories/comparable_properties/test_epc_comparable_properties_repository.py b/tests/repositories/comparable_properties/test_epc_comparable_properties_repository.py index 1a0b918ea..5b99bbd41 100644 --- a/tests/repositories/comparable_properties/test_epc_comparable_properties_repository.py +++ b/tests/repositories/comparable_properties/test_epc_comparable_properties_repository.py @@ -245,6 +245,83 @@ def test_candidates_near_without_a_source_uses_only_the_seed() -> None: assert [c.certificate_number for c in candidates] == ["CERT-1"] +def test_candidates_near_widens_once_when_the_normal_reach_is_insufficient() -> None: + # Arrange — the normal-radius nearby set (P0, P1) has no MATCH cert; the + # wider-radius set (P0, P1, P2, P3) reaches P3, which does. + client = _MultiPostcodeEpcClient( + { + "P0": [_result("OTHER-1", uprn=1)], + "P1": [_result("OTHER-2", uprn=2)], + "P2": [_result("OTHER-3", uprn=3)], + "P3": [_result("MATCH-1", uprn=4)], + } + ) + normal = _FakeNearbyPostcodes(["P0", "P1"]) + wider = _FakeNearbyPostcodes(["P0", "P1", "P2", "P3"]) + repo = EpcComparablePropertiesRepository( + client, + _FakeGeospatial({}), + nearby_postcodes=normal, + widen_nearby_postcodes=wider, + ) + + # Act + candidates = repo.candidates_near( + "P0", None, enough=lambda c: c.certificate_number.startswith("MATCH"), minimum=1 + ) + + # Assert — both the normal and the widened source were consulted (in that + # order), and the wider walk's matches are returned. + assert normal.calls == [("P0", None)] + assert wider.calls == [("P0", None)] + certs = {c.certificate_number for c in candidates} + assert "MATCH-1" in certs + + +def test_candidates_near_does_not_widen_when_the_normal_reach_is_enough() -> None: + # Arrange — the normal radius already yields the minimum match, so the + # wider (more expensive) source must never be consulted. + client = _MultiPostcodeEpcClient({"P0": [_result("MATCH-1", uprn=1)]}) + normal = _FakeNearbyPostcodes(["P0"]) + wider = _FakeNearbyPostcodes(["P0", "P1", "P2"]) + repo = EpcComparablePropertiesRepository( + client, + _FakeGeospatial({}), + nearby_postcodes=normal, + widen_nearby_postcodes=wider, + ) + + # Act + candidates = repo.candidates_near( + "P0", None, enough=lambda c: c.certificate_number.startswith("MATCH"), minimum=1 + ) + + # Assert + assert wider.calls == [] + assert [c.certificate_number for c in candidates] == ["MATCH-1"] + + +def test_candidates_near_without_enough_predicate_never_widens() -> None: + # Arrange — no `enough` predicate means the caller isn't gating on a match + # count (e.g. the unconditional aggregate path), so widening is meaningless + # and must not fire even with a configured wider source. + client = _MultiPostcodeEpcClient({"P0": [_result("CERT-1", uprn=1)]}) + normal = _FakeNearbyPostcodes(["P0"]) + wider = _FakeNearbyPostcodes(["P0", "P1"]) + repo = EpcComparablePropertiesRepository( + client, + _FakeGeospatial({}), + nearby_postcodes=normal, + widen_nearby_postcodes=wider, + ) + + # Act + repo.candidates_near("P0", None) + + # Assert + assert wider.calls == [] + + # --------------------------------------------------------------------------- # Unmappable cohort certs are skipped + recorded, not allowed to sink the cohort # --------------------------------------------------------------------------- diff --git a/tests/repositories/property/test_landlord_address_resolver.py b/tests/repositories/property/test_landlord_address_resolver.py new file mode 100644 index 000000000..4bc67a76f --- /dev/null +++ b/tests/repositories/property/test_landlord_address_resolver.py @@ -0,0 +1,39 @@ +from collections.abc import Iterator + +import pytest +from sqlalchemy import Engine +from sqlmodel import Session + +from backend.app.db.models.hubspot_deal_data import HubspotDealData +from repositories.property.landlord_address_resolver import LandlordAddressResolver + + +@pytest.fixture +def session(db_engine: Engine) -> Iterator[Session]: + with Session(db_engine) as s: + yield s + + +def test_resolves_deal_names_and_falls_back_for_unknowns( + session: Session, +) -> None: + # arrange — two properties with deal names; a third id has no deal. + session.add( + HubspotDealData( + deal_id="d1", landlord_property_id="LP1", dealname="12 Oak Street" + ) + ) + session.add( + HubspotDealData( + deal_id="d2", landlord_property_id="LP2", dealname="9 Elm Road" + ) + ) + session.flush() + + # act + resolver = LandlordAddressResolver(session, ["LP1", "LP2", "LP3"]) + + # assert — the deal name becomes the folder label; unknowns fall back. + assert resolver.address_for("LP1") == "12 Oak Street" + assert resolver.address_for("LP2") == "9 Elm Road" + assert resolver.address_for("LP3") == "address unavailable" diff --git a/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py b/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py index 5300c0209..76ac9b758 100644 --- a/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py +++ b/tests/repositories/uploaded_file/test_uploaded_file_postgres_repository.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import Engine from sqlmodel import Session +from backend.app.db.models.hubspot_deal_data import HubspotDealData from infrastructure.postgres.uploaded_file_table import FileTypeEnum, UploadedFile from repositories.uploaded_file.uploaded_file_postgres_repository import ( UploadedFilePostgresRepository, @@ -78,3 +79,66 @@ def test_does_not_return_row_with_different_file_type(db_engine: Engine) -> None # Assert assert result is None + + +def _deal_file( + hubspot_deal_id: str, + s3_file_key: str, + file_type: FileTypeEnum = FileTypeEnum.SITE_NOTE, +) -> UploadedFile: + # A real uploaded file: keyed by hubspot_deal_id, landlord_property_id NULL + # (no upload source populates it — see ADR-0060 matching decision). + return UploadedFile( + s3_file_bucket=_BUCKET, + s3_file_key=s3_file_key, + s3_upload_timestamp=datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + hubspot_deal_id=hubspot_deal_id, + file_type=file_type.value, + ) + + +def _deal(landlord_property_id: str, deal_id: str) -> HubspotDealData: + return HubspotDealData(deal_id=deal_id, landlord_property_id=landlord_property_id) + + +def test_by_landlord_property_ids_bridges_deal_id_to_the_property( + db_engine: Engine, +) -> None: + # Arrange — files are keyed by hubspot_deal_id; hubspot_deal_data bridges each + # deal to a landlord_property_id. Three properties exist; two are requested. + with Session(db_engine) as session: + session.add(_deal("LP1", "deal-1")) + session.add(_deal("LP2", "deal-2")) + session.add(_deal("LP3", "deal-3")) + session.add(_deal_file("deal-1", "one.pdf")) + session.add(_deal_file("deal-2", "two.pdf")) + session.add(_deal_file("deal-3", "three.pdf")) + session.commit() + + # Act + with Session(db_engine) as session: + found = UploadedFilePostgresRepository(session).by_landlord_property_ids( + ["LP1", "LP2"] + ) + result = {(d.landlord_property_id, d.s3_key) for d in found} + + # Assert — the file's property comes from the bridge, keyed by the deal. + assert result == {("LP1", "one.pdf"), ("LP2", "two.pdf")} + + +def test_by_landlord_property_ids_matches_nothing_without_a_deal_bridge( + db_engine: Engine, +) -> None: + # Arrange — a file whose deal has no hubspot_deal_data row can't be reached. + with Session(db_engine) as session: + session.add(_deal_file("orphan-deal", "orphan.pdf")) + session.commit() + + # Act + with Session(db_engine) as session: + found = UploadedFilePostgresRepository(session).by_landlord_property_ids( + ["LP1"] + ) + + # Assert + assert found == []