mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
Compare commits
6 commits
94cbf5f516
...
0dee917094
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0dee917094 | ||
|
|
91bb4b6571 | ||
|
|
84098e28ff | ||
|
|
5b677dedbe | ||
|
|
cf14a4e3aa | ||
|
|
acb306f7b9 |
26 changed files with 317 additions and 309 deletions
|
|
@ -1,13 +1,13 @@
|
|||
from typing import Any
|
||||
import boto3
|
||||
from orchestration.landlord_description_overrides_orchestrator import (
|
||||
LandlordDescriptionOverridesOrchestrator,
|
||||
from orchestration.sal_orchestrator import (
|
||||
SALOrchestrator,
|
||||
)
|
||||
from infrastructure.csv_s3_client import CsvS3Client
|
||||
from repositories.user_address.user_address_csv_s3_repository import (
|
||||
UserAddressCsvS3Repository,
|
||||
from repositories.unsanitised_address.unsanitised_address_list_csv_s3_repository import (
|
||||
UnsanitisedAddressListCsvS3Repository,
|
||||
)
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList
|
||||
|
||||
|
||||
def handler(
|
||||
|
|
@ -20,24 +20,20 @@ def handler(
|
|||
|
||||
# boto3.client is overloaded per-service in the installed stubs; cast
|
||||
# to Any so the strict-mode checker treats it as opaque.
|
||||
boto3_client: Any = (
|
||||
boto3.client
|
||||
) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
||||
boto3_client: Any = boto3.client # noqa
|
||||
boto_s3: Any = boto3_client("s3")
|
||||
|
||||
csv_client = CsvS3Client(boto_s3, bucket)
|
||||
user_address_repo = UserAddressCsvS3Repository(csv_client, bucket)
|
||||
unsanitised_address_repo = UnsanitisedAddressListCsvS3Repository(csv_client, bucket)
|
||||
|
||||
orchestrator = LandlordDescriptionOverridesOrchestrator(
|
||||
user_address_repo=user_address_repo,
|
||||
sal = SALOrchestrator(
|
||||
unsanitised_address_repo=unsanitised_address_repo,
|
||||
)
|
||||
|
||||
list_of_user_address: list[LandlordAssetList] = orchestrator.get_user_address(
|
||||
input_s3_uri=s3_uri
|
||||
)
|
||||
addressList: AddressList = sal.get_unsanitised_addresses(input_s3_uri=s3_uri)
|
||||
|
||||
col_to_desc_map = orchestrator.get_col_to_description_mappings(
|
||||
list_of_user_address=list_of_user_address
|
||||
col_to_desc_map = sal.get_col_to_description_mappings(
|
||||
list_of_unsanitised_address=addressList
|
||||
)
|
||||
|
||||
# Read csv of user input
|
||||
|
|
@ -12,8 +12,8 @@ from infrastructure.address2uprn_queue_client import Address2UprnQueueClient
|
|||
from infrastructure.csv_s3_client import CsvS3Client
|
||||
from orchestration.postcode_splitter_orchestrator import PostcodeSplitterOrchestrator
|
||||
from orchestration.task_orchestrator import TaskOrchestrator
|
||||
from repositories.user_address.user_address_csv_s3_repository import (
|
||||
UserAddressCsvS3Repository,
|
||||
from repositories.unsanitised_address.unsanitised_address_list_csv_s3_repository import (
|
||||
UnsanitisedAddressListCsvS3Repository,
|
||||
)
|
||||
from utilities.aws_lambda.subtask_handler import subtask_handler
|
||||
|
||||
|
|
@ -29,17 +29,19 @@ def handler(
|
|||
|
||||
# boto3.client is overloaded per-service in the installed stubs; cast
|
||||
# to Any so the strict-mode checker treats it as opaque.
|
||||
boto3_client: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
||||
boto3_client: Any = (
|
||||
boto3.client
|
||||
) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
||||
boto_s3: Any = boto3_client("s3")
|
||||
boto_sqs: Any = boto3_client("sqs")
|
||||
|
||||
csv_client = CsvS3Client(boto_s3, bucket)
|
||||
user_address_repo = UserAddressCsvS3Repository(csv_client, bucket)
|
||||
unsanitised_address_repo = UnsanitisedAddressListCsvS3Repository(csv_client, bucket)
|
||||
queue_client = Address2UprnQueueClient(boto_sqs, queue_url)
|
||||
|
||||
splitter = PostcodeSplitterOrchestrator(
|
||||
task_orchestrator=task_orchestrator,
|
||||
user_address_repo=user_address_repo,
|
||||
unsanitised_address_repo=unsanitised_address_repo,
|
||||
queue_client=queue_client,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,21 +2,21 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList, UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def iter_postcode_grouped_batches(
|
||||
addresses: Iterable[LandlordAssetList],
|
||||
addresses: Iterable[UnsanitisedAddress],
|
||||
*,
|
||||
max_batch_size: int = 500,
|
||||
) -> Iterator[list[LandlordAssetList]]:
|
||||
) -> Iterator[AddressList]:
|
||||
if max_batch_size < 1:
|
||||
raise ValueError("max_batch_size must be >= 1")
|
||||
|
||||
groups = _group_by_postcode_in_order(addresses)
|
||||
|
||||
buffer: list[LandlordAssetList] = []
|
||||
buffer: AddressList = AddressList([])
|
||||
for group in groups.values():
|
||||
group_len = len(group)
|
||||
|
||||
|
|
@ -26,14 +26,14 @@ def iter_postcode_grouped_batches(
|
|||
if group_len >= max_batch_size:
|
||||
if buffer:
|
||||
yield buffer
|
||||
buffer = []
|
||||
buffer = AddressList([])
|
||||
yield group
|
||||
continue
|
||||
|
||||
# Adding this group would overflow: flush buffer before appending.
|
||||
if len(buffer) + group_len > max_batch_size:
|
||||
yield buffer
|
||||
buffer = []
|
||||
buffer = AddressList([])
|
||||
|
||||
buffer.extend(group)
|
||||
|
||||
|
|
@ -43,9 +43,9 @@ def iter_postcode_grouped_batches(
|
|||
|
||||
|
||||
def _group_by_postcode_in_order(
|
||||
addresses: Iterable[LandlordAssetList],
|
||||
) -> dict[Postcode, list[LandlordAssetList]]:
|
||||
groups: dict[Postcode, list[LandlordAssetList]] = {}
|
||||
addresses: Iterable[UnsanitisedAddress],
|
||||
) -> dict[Postcode, AddressList]:
|
||||
groups: dict[Postcode, AddressList] = {}
|
||||
for address in addresses:
|
||||
groups.setdefault(address.postcode, []).append(address)
|
||||
groups.setdefault(address.postcode, AddressList([])).append(address)
|
||||
return groups
|
||||
|
|
|
|||
24
domain/addresses/unsanitised_address.py
Normal file
24
domain/addresses/unsanitised_address.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, NewType
|
||||
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def _empty_source_row() -> dict[str, str]:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnsanitisedAddress:
|
||||
address: str
|
||||
postcode: Postcode
|
||||
org_reference: Optional[str] = None
|
||||
additional_info: dict[str, str] = field(
|
||||
default_factory=_empty_source_row, compare=False
|
||||
)
|
||||
|
||||
|
||||
# A batch of raw, pre-standardisation addresses as supplied by a landlord.
|
||||
AddressList = NewType("AddressList", list[UnsanitisedAddress])
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def _empty_source_row() -> dict[str, str]:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LandlordAssetList:
|
||||
user_address: str
|
||||
postcode: Postcode
|
||||
internal_reference: Optional[str] = None
|
||||
landlord_additional_info: dict[str, str] = field(
|
||||
default_factory=_empty_source_row, compare=False
|
||||
)
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from repositories.user_address.user_address_repository import UserAddressRepository
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
|
||||
|
||||
class LandlordDescriptionOverridesOrchestrator:
|
||||
def __init__(self, user_address_repo: UserAddressRepository) -> None:
|
||||
self._user_address_repo = user_address_repo
|
||||
|
||||
def get_user_address(
|
||||
self,
|
||||
input_s3_uri: str,
|
||||
) -> list[LandlordAssetList]:
|
||||
return self._user_address_repo.load_batch(input_s3_uri)
|
||||
|
||||
def get_col_to_description_mappings(
|
||||
self, list_of_user_address: list[LandlordAssetList]
|
||||
) -> dict[str, set[str]]:
|
||||
mappings: dict[str, set[str]] = {}
|
||||
for user_address in list_of_user_address:
|
||||
for key, value in user_address.landlord_additional_info.items():
|
||||
# Lower-case so case-only typos collapse to one variant.
|
||||
mappings.setdefault(key, set()).add(value.lower())
|
||||
return mappings
|
||||
|
|
@ -5,19 +5,21 @@ from uuid import UUID
|
|||
from infrastructure.address2uprn_queue_client import Address2UprnQueueClient
|
||||
from orchestration.task_orchestrator import TaskOrchestrator
|
||||
from domain.addresses.postcode_batching import iter_postcode_grouped_batches
|
||||
from repositories.user_address.user_address_repository import UserAddressRepository
|
||||
from repositories.unsanitised_address.unsanitised_address_list_repository import (
|
||||
UnsanitisedAddressListRepository,
|
||||
)
|
||||
|
||||
|
||||
class PostcodeSplitterOrchestrator:
|
||||
def __init__(
|
||||
self,
|
||||
task_orchestrator: TaskOrchestrator,
|
||||
user_address_repo: UserAddressRepository,
|
||||
unsanitised_address_repo: UnsanitisedAddressListRepository,
|
||||
queue_client: Address2UprnQueueClient,
|
||||
max_batch_size: int = 500,
|
||||
) -> None:
|
||||
self._task_orchestrator = task_orchestrator
|
||||
self._user_address_repo = user_address_repo
|
||||
self._unsanitised_address_repo = unsanitised_address_repo
|
||||
self._queue_client = queue_client
|
||||
self._max_batch_size = max_batch_size
|
||||
|
||||
|
|
@ -28,7 +30,7 @@ class PostcodeSplitterOrchestrator:
|
|||
parent_subtask_id: UUID,
|
||||
input_s3_uri: str,
|
||||
) -> list[UUID]:
|
||||
addresses = self._user_address_repo.load_batch(input_s3_uri)
|
||||
addresses = self._unsanitised_address_repo.load_batch(input_s3_uri)
|
||||
path_prefix = (
|
||||
f"ara_postcode_splitter_batches/{parent_task_id}/{parent_subtask_id}"
|
||||
)
|
||||
|
|
@ -37,7 +39,7 @@ class PostcodeSplitterOrchestrator:
|
|||
for batch in iter_postcode_grouped_batches(
|
||||
addresses, max_batch_size=self._max_batch_size
|
||||
):
|
||||
batch_uri = self._user_address_repo.save_batch(batch, path_prefix)
|
||||
batch_uri = self._unsanitised_address_repo.save_batch(batch, path_prefix)
|
||||
child = self._task_orchestrator.create_child_subtask(
|
||||
parent_task_id,
|
||||
inputs={
|
||||
|
|
|
|||
25
orchestration/sal_orchestrator.py
Normal file
25
orchestration/sal_orchestrator.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from repositories.unsanitised_address.unsanitised_address_list_repository import (
|
||||
UnsanitisedAddressListRepository,
|
||||
)
|
||||
from domain.addresses.unsanitised_address import AddressList
|
||||
|
||||
|
||||
class SALOrchestrator:
|
||||
def __init__(self, unsanitised_address_repo: UnsanitisedAddressListRepository) -> None:
|
||||
self._unsanitised_address_repo = unsanitised_address_repo
|
||||
|
||||
def get_unsanitised_addresses(
|
||||
self,
|
||||
input_s3_uri: str,
|
||||
) -> AddressList:
|
||||
return self._unsanitised_address_repo.load_batch(input_s3_uri)
|
||||
|
||||
def get_col_to_description_mappings(
|
||||
self, list_of_unsanitised_address: AddressList
|
||||
) -> dict[str, set[str]]:
|
||||
mappings: dict[str, set[str]] = {}
|
||||
for unsanitised_address in list_of_unsanitised_address:
|
||||
for key, value in unsanitised_address.additional_info.items():
|
||||
# Lower-case so case-only typos collapse to one variant.
|
||||
mappings.setdefault(key, set()).add(value.lower())
|
||||
return mappings
|
||||
|
|
@ -4,10 +4,12 @@ import uuid
|
|||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList, UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
from infrastructure.csv_s3_client import CsvS3Client
|
||||
from repositories.user_address.user_address_repository import UserAddressRepository
|
||||
from repositories.unsanitised_address.unsanitised_address_list_repository import (
|
||||
UnsanitisedAddressListRepository,
|
||||
)
|
||||
|
||||
_ADDRESS_COLUMNS: tuple[str, str, str] = ("Address 1", "Address 2", "Address 3")
|
||||
_POSTCODE_COLUMN: str = "postcode"
|
||||
|
|
@ -15,43 +17,43 @@ _INTERNAL_REFERENCE_COLUMN: str = "Internal Reference"
|
|||
_POSTCODE_CLEAN_COLUMN: str = "postcode_clean"
|
||||
|
||||
|
||||
class UserAddressCsvS3Repository(UserAddressRepository):
|
||||
class UnsanitisedAddressListCsvS3Repository(UnsanitisedAddressListRepository):
|
||||
def __init__(self, csv_client: CsvS3Client, bucket: str) -> None:
|
||||
self._csv_client = csv_client
|
||||
self._bucket = bucket
|
||||
|
||||
def load_batch(self, s3_uri: str) -> list[LandlordAssetList]:
|
||||
def load_batch(self, s3_uri: str) -> AddressList:
|
||||
rows = self._csv_client.read_rows(s3_uri)
|
||||
if rows and _POSTCODE_COLUMN not in rows[0]:
|
||||
raise ValueError(
|
||||
f"Input CSV {s3_uri} has no {_POSTCODE_COLUMN!r} column; "
|
||||
f"columns present: {sorted(rows[0])}"
|
||||
)
|
||||
addresses: list[LandlordAssetList] = []
|
||||
addresses: AddressList = AddressList([])
|
||||
for row in rows:
|
||||
parts = [
|
||||
row[col].strip()
|
||||
for col in _ADDRESS_COLUMNS
|
||||
if col in row and row[col].strip()
|
||||
]
|
||||
user_address = ", ".join(parts)
|
||||
unsanitised_address = ", ".join(parts)
|
||||
postcode = row.get(_POSTCODE_COLUMN, "")
|
||||
raw_ref = row.get(_INTERNAL_REFERENCE_COLUMN, "").strip()
|
||||
internal_reference: Optional[str] = raw_ref or None
|
||||
addresses.append(
|
||||
LandlordAssetList(
|
||||
user_address=user_address,
|
||||
UnsanitisedAddress(
|
||||
address=unsanitised_address,
|
||||
postcode=Postcode(postcode),
|
||||
internal_reference=internal_reference,
|
||||
landlord_additional_info=row,
|
||||
org_reference=internal_reference,
|
||||
additional_info=row,
|
||||
)
|
||||
)
|
||||
return addresses
|
||||
|
||||
def save_batch(self, addresses: list[LandlordAssetList], path_prefix: str) -> str:
|
||||
def save_batch(self, addresses: AddressList, path_prefix: str) -> str:
|
||||
rows: list[dict[str, str]] = [
|
||||
{
|
||||
**addr.landlord_additional_info,
|
||||
**addr.additional_info,
|
||||
_POSTCODE_CLEAN_COLUMN: str(addr.postcode),
|
||||
}
|
||||
for addr in addresses
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from domain.addresses.unsanitised_address import AddressList
|
||||
|
||||
|
||||
class UnsanitisedAddressListRepository(ABC):
|
||||
@abstractmethod
|
||||
def load_batch(self, s3_uri: str) -> AddressList: ...
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(self, addresses: AddressList, path_prefix: str) -> str: ...
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
|
||||
|
||||
class UserAddressRepository(ABC):
|
||||
@abstractmethod
|
||||
def load_batch(self, s3_uri: str) -> list[LandlordAssetList]: ...
|
||||
|
||||
@abstractmethod
|
||||
def save_batch(
|
||||
self, addresses: list[LandlordAssetList], path_prefix: str
|
||||
) -> str: ...
|
||||
|
|
@ -1,17 +1,17 @@
|
|||
import pytest
|
||||
|
||||
from domain.addresses.postcode_batching import iter_postcode_grouped_batches
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList, UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def _addrs(postcode: str, n: int) -> list[LandlordAssetList]:
|
||||
return [
|
||||
LandlordAssetList(
|
||||
user_address=f"{i} {postcode} Street", postcode=Postcode(postcode)
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
def _addrs(postcode: str, n: int) -> AddressList:
|
||||
return AddressList(
|
||||
[
|
||||
UnsanitisedAddress(address=f"{i} {postcode} Street", postcode=Postcode(postcode))
|
||||
for i in range(n)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_empty_input_yields_no_batches() -> None:
|
||||
|
|
|
|||
96
tests/domain/addresses/test_unsanitised_address.py
Normal file
96
tests/domain/addresses/test_unsanitised_address.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from domain.addresses.unsanitised_address import UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def test_unsanitised_address_holds_postcode_value_object() -> None:
|
||||
# act
|
||||
addr = UnsanitisedAddress(address="1 The Street", postcode=Postcode("sw1a 1aa"))
|
||||
# assert
|
||||
assert addr.postcode == Postcode("SW1A1AA")
|
||||
|
||||
|
||||
def test_unsanitised_address_preserves_unsanitised_address_verbatim() -> None:
|
||||
# The free-text unsanitised_address string is intentionally NOT normalised --
|
||||
# only the postcode is canonicalised, and that happens inside Postcode.
|
||||
# act
|
||||
addr = UnsanitisedAddress(address=" 1 The Street ", postcode=Postcode("SW1A1AA"))
|
||||
# assert
|
||||
assert addr.address == " 1 The Street "
|
||||
|
||||
|
||||
def test_unsanitised_address_internal_reference_defaults_to_none() -> None:
|
||||
# act
|
||||
addr = UnsanitisedAddress(address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# assert
|
||||
assert addr.org_reference is None
|
||||
|
||||
|
||||
def test_unsanitised_address_internal_reference_accepted() -> None:
|
||||
# act
|
||||
addr = UnsanitisedAddress(
|
||||
address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
org_reference="cust-42",
|
||||
)
|
||||
# assert
|
||||
assert addr.org_reference == "cust-42"
|
||||
|
||||
|
||||
def test_unsanitised_address_is_frozen() -> None:
|
||||
# arrange
|
||||
addr = UnsanitisedAddress(address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# act / assert
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
addr.postcode = Postcode("OTHER") # type: ignore[misc]
|
||||
|
||||
|
||||
def test_unsanitised_address_equality_uses_canonical_postcode() -> None:
|
||||
# Postcode sanitises eagerly, so addresses built from different surface
|
||||
# forms of the same postcode compare equal.
|
||||
# arrange
|
||||
a = UnsanitisedAddress(address="1 The Street", postcode=Postcode("sw1a 1aa"))
|
||||
b = UnsanitisedAddress(address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# act / assert
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_unsanitised_address_source_row_defaults_to_empty_dict() -> None:
|
||||
# act
|
||||
addr = UnsanitisedAddress(address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# assert
|
||||
assert addr.additional_info == {}
|
||||
|
||||
|
||||
def test_unsanitised_address_carries_source_row() -> None:
|
||||
# arrange
|
||||
row = {"Address 1": "1 The Street", "postcode": "SW1A 1AA", "SAP Score": "72"}
|
||||
# act
|
||||
addr = UnsanitisedAddress(
|
||||
address="1 The Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
additional_info=row,
|
||||
)
|
||||
# assert
|
||||
assert addr.additional_info == row
|
||||
|
||||
|
||||
def test_unsanitised_address_equality_ignores_source_row() -> None:
|
||||
# source_row is excluded from equality (and hashing): identity stays
|
||||
# defined by the parsed fields.
|
||||
# arrange
|
||||
a = UnsanitisedAddress(
|
||||
address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
additional_info={"x": "1"},
|
||||
)
|
||||
b = UnsanitisedAddress(
|
||||
address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
additional_info={"y": "2"},
|
||||
)
|
||||
# act / assert
|
||||
assert a == b
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.postcode import Postcode
|
||||
|
||||
|
||||
def test_user_address_holds_postcode_value_object() -> None:
|
||||
# act
|
||||
addr = LandlordAssetList(user_address="1 The Street", postcode=Postcode("sw1a 1aa"))
|
||||
# assert
|
||||
assert addr.postcode == Postcode("SW1A1AA")
|
||||
|
||||
|
||||
def test_user_address_preserves_user_address_verbatim() -> None:
|
||||
# The free-text user_address string is intentionally NOT normalised --
|
||||
# only the postcode is canonicalised, and that happens inside Postcode.
|
||||
# act
|
||||
addr = LandlordAssetList(
|
||||
user_address=" 1 The Street ", postcode=Postcode("SW1A1AA")
|
||||
)
|
||||
# assert
|
||||
assert addr.user_address == " 1 The Street "
|
||||
|
||||
|
||||
def test_user_address_internal_reference_defaults_to_none() -> None:
|
||||
# act
|
||||
addr = LandlordAssetList(user_address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# assert
|
||||
assert addr.internal_reference is None
|
||||
|
||||
|
||||
def test_user_address_internal_reference_accepted() -> None:
|
||||
# act
|
||||
addr = LandlordAssetList(
|
||||
user_address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
internal_reference="cust-42",
|
||||
)
|
||||
# assert
|
||||
assert addr.internal_reference == "cust-42"
|
||||
|
||||
|
||||
def test_user_address_is_frozen() -> None:
|
||||
# arrange
|
||||
addr = LandlordAssetList(user_address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# act / assert
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
addr.postcode = Postcode("OTHER") # type: ignore[misc]
|
||||
|
||||
|
||||
def test_user_address_equality_uses_canonical_postcode() -> None:
|
||||
# Postcode sanitises eagerly, so addresses built from different surface
|
||||
# forms of the same postcode compare equal.
|
||||
# arrange
|
||||
a = LandlordAssetList(user_address="1 The Street", postcode=Postcode("sw1a 1aa"))
|
||||
b = LandlordAssetList(user_address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# act / assert
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_user_address_source_row_defaults_to_empty_dict() -> None:
|
||||
# act
|
||||
addr = LandlordAssetList(user_address="1 The Street", postcode=Postcode("SW1A1AA"))
|
||||
# assert
|
||||
assert addr.landlord_additional_info == {}
|
||||
|
||||
|
||||
def test_user_address_carries_source_row() -> None:
|
||||
# arrange
|
||||
row = {"Address 1": "1 The Street", "postcode": "SW1A 1AA", "SAP Score": "72"}
|
||||
# act
|
||||
addr = LandlordAssetList(
|
||||
user_address="1 The Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
landlord_additional_info=row,
|
||||
)
|
||||
# assert
|
||||
assert addr.landlord_additional_info == row
|
||||
|
||||
|
||||
def test_user_address_equality_ignores_source_row() -> None:
|
||||
# source_row is excluded from equality (and hashing): identity stays
|
||||
# defined by the parsed fields.
|
||||
# arrange
|
||||
a = LandlordAssetList(
|
||||
user_address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
landlord_additional_info={"x": "1"},
|
||||
)
|
||||
b = LandlordAssetList(
|
||||
user_address="1 The Street",
|
||||
postcode=Postcode("SW1A1AA"),
|
||||
landlord_additional_info={"y": "2"},
|
||||
)
|
||||
# act / assert
|
||||
assert a == b
|
||||
|
|
@ -1,44 +1,46 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList, UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
from orchestration.landlord_description_overrides_orchestrator import (
|
||||
LandlordDescriptionOverridesOrchestrator,
|
||||
from orchestration.sal_orchestrator import (
|
||||
SALOrchestrator,
|
||||
)
|
||||
from repositories.unsanitised_address.unsanitised_address_list_repository import (
|
||||
UnsanitisedAddressListRepository,
|
||||
)
|
||||
from repositories.user_address.user_address_repository import UserAddressRepository
|
||||
|
||||
|
||||
class _StubUserAddressRepository(UserAddressRepository):
|
||||
class _StubUnsanitisedAddressRepository(UnsanitisedAddressListRepository):
|
||||
"""``get_col_to_description_mappings`` never touches the repo."""
|
||||
|
||||
def load_batch(self, s3_uri: str) -> list[LandlordAssetList]:
|
||||
def load_batch(self, s3_uri: str) -> AddressList:
|
||||
raise NotImplementedError()
|
||||
|
||||
def save_batch(self, addresses: list[LandlordAssetList], path_prefix: str) -> str:
|
||||
def save_batch(self, addresses: AddressList, path_prefix: str) -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def _make_user_address(landlord_additional_info: dict[str, str]) -> LandlordAssetList:
|
||||
return LandlordAssetList(
|
||||
user_address="1 High St",
|
||||
def _make_unsanitised_address(landlord_additional_info: dict[str, str]) -> UnsanitisedAddress:
|
||||
return UnsanitisedAddress(
|
||||
address="1 High St",
|
||||
postcode=Postcode("AA1 1AA"),
|
||||
landlord_additional_info=landlord_additional_info,
|
||||
additional_info=landlord_additional_info,
|
||||
)
|
||||
|
||||
|
||||
def _orchestrator() -> LandlordDescriptionOverridesOrchestrator:
|
||||
return LandlordDescriptionOverridesOrchestrator(
|
||||
user_address_repo=_StubUserAddressRepository()
|
||||
)
|
||||
def _orchestrator() -> SALOrchestrator:
|
||||
return SALOrchestrator(unsanitised_address_repo=_StubUnsanitisedAddressRepository())
|
||||
|
||||
|
||||
def test_collects_every_value_per_shared_key() -> None:
|
||||
# arrange: every address carries the same keys, all values distinct.
|
||||
addresses = [
|
||||
_make_user_address({"description": "cosy", "condition": "new"}),
|
||||
_make_user_address({"description": "spacious", "condition": "worn"}),
|
||||
_make_user_address({"description": "bright", "condition": "fair"}),
|
||||
]
|
||||
addresses = AddressList(
|
||||
[
|
||||
_make_unsanitised_address({"description": "cosy", "condition": "new"}),
|
||||
_make_unsanitised_address({"description": "spacious", "condition": "worn"}),
|
||||
_make_unsanitised_address({"description": "bright", "condition": "fair"}),
|
||||
]
|
||||
)
|
||||
|
||||
# act
|
||||
mappings = _orchestrator().get_col_to_description_mappings(addresses)
|
||||
|
|
@ -52,11 +54,13 @@ def test_collects_every_value_per_shared_key() -> None:
|
|||
|
||||
def test_repeated_values_collapse_to_one_variant() -> None:
|
||||
# arrange: two addresses share the same wall description.
|
||||
addresses = [
|
||||
_make_user_address({"description": "cosy"}),
|
||||
_make_user_address({"description": "cosy"}),
|
||||
_make_user_address({"description": "bright"}),
|
||||
]
|
||||
addresses = AddressList(
|
||||
[
|
||||
_make_unsanitised_address({"description": "cosy"}),
|
||||
_make_unsanitised_address({"description": "cosy"}),
|
||||
_make_unsanitised_address({"description": "bright"}),
|
||||
]
|
||||
)
|
||||
|
||||
# act
|
||||
mappings = _orchestrator().get_col_to_description_mappings(addresses)
|
||||
|
|
@ -67,11 +71,13 @@ def test_repeated_values_collapse_to_one_variant() -> None:
|
|||
|
||||
def test_case_only_variants_collapse_to_one() -> None:
|
||||
# arrange: the same description typed with inconsistent casing.
|
||||
addresses = [
|
||||
_make_user_address({"description": "Cosy"}),
|
||||
_make_user_address({"description": "cosy"}),
|
||||
_make_user_address({"description": "COSY"}),
|
||||
]
|
||||
addresses = AddressList(
|
||||
[
|
||||
_make_unsanitised_address({"description": "Cosy"}),
|
||||
_make_unsanitised_address({"description": "cosy"}),
|
||||
_make_unsanitised_address({"description": "COSY"}),
|
||||
]
|
||||
)
|
||||
|
||||
# act
|
||||
mappings = _orchestrator().get_col_to_description_mappings(addresses)
|
||||
|
|
@ -82,7 +88,7 @@ def test_case_only_variants_collapse_to_one() -> None:
|
|||
|
||||
def test_empty_address_list_yields_empty_mapping() -> None:
|
||||
# arrange / act
|
||||
mappings = _orchestrator().get_col_to_description_mappings([])
|
||||
mappings = _orchestrator().get_col_to_description_mappings(AddressList([]))
|
||||
|
||||
# assert
|
||||
assert mappings == {}
|
||||
|
|
@ -90,7 +96,7 @@ def test_empty_address_list_yields_empty_mapping() -> None:
|
|||
|
||||
def test_single_address_yields_single_value_per_key() -> None:
|
||||
# arrange
|
||||
addresses = [_make_user_address({"description": "cosy"})]
|
||||
addresses = AddressList([_make_unsanitised_address({"description": "cosy"})])
|
||||
|
||||
# act
|
||||
mappings = _orchestrator().get_col_to_description_mappings(addresses)
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ from orchestration.postcode_splitter_orchestrator import PostcodeSplitterOrchest
|
|||
from orchestration.task_orchestrator import TaskOrchestrator
|
||||
from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository
|
||||
from repositories.tasks.task_postgres_repository import TaskPostgresRepository
|
||||
from repositories.user_address.user_address_csv_s3_repository import (
|
||||
UserAddressCsvS3Repository,
|
||||
from repositories.unsanitised_address.unsanitised_address_list_csv_s3_repository import (
|
||||
UnsanitisedAddressListCsvS3Repository,
|
||||
)
|
||||
|
||||
BUCKET = "splitter-bucket"
|
||||
|
|
@ -27,7 +27,9 @@ REGION = "us-east-1"
|
|||
|
||||
|
||||
def _make_boto_client(service_name: str) -> Any:
|
||||
factory: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
||||
factory: Any = (
|
||||
boto3.client
|
||||
) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
||||
return factory(service_name, region_name=REGION)
|
||||
|
||||
|
||||
|
|
@ -62,7 +64,7 @@ class Harness:
|
|||
csv_client: CsvS3Client
|
||||
boto_sqs: Any
|
||||
queue_url: str
|
||||
repo: UserAddressCsvS3Repository
|
||||
repo: UnsanitisedAddressListCsvS3Repository
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -76,7 +78,7 @@ def harness(db_engine: Engine) -> Iterator[Harness]:
|
|||
queue_url = cast(str, queue["QueueUrl"])
|
||||
|
||||
csv_client = CsvS3Client(boto_s3, BUCKET)
|
||||
repo = UserAddressCsvS3Repository(csv_client, BUCKET)
|
||||
repo = UnsanitisedAddressListCsvS3Repository(csv_client, BUCKET)
|
||||
queue_client = Address2UprnQueueClient(boto_sqs, queue_url)
|
||||
|
||||
# DB: ephemeral PostgreSQL TaskOrchestrator
|
||||
|
|
@ -89,7 +91,7 @@ def harness(db_engine: Engine) -> Iterator[Harness]:
|
|||
|
||||
splitter = PostcodeSplitterOrchestrator(
|
||||
task_orchestrator=task_orchestrator,
|
||||
user_address_repo=repo,
|
||||
unsanitised_address_repo=repo,
|
||||
queue_client=queue_client,
|
||||
max_batch_size=3,
|
||||
)
|
||||
|
|
@ -169,10 +171,8 @@ def test_split_and_dispatch_creates_three_children_for_fixture(
|
|||
harness: Harness,
|
||||
) -> None:
|
||||
# arrange
|
||||
parent_task, parent_subtask = (
|
||||
harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
parent_task, parent_subtask = harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
input_uri = _upload_fixture_csv(harness.csv_client)
|
||||
|
||||
|
|
@ -197,10 +197,8 @@ def test_split_and_dispatch_persists_child_inputs_with_task_id_and_s3_uri(
|
|||
harness: Harness,
|
||||
) -> None:
|
||||
# arrange
|
||||
parent_task, parent_subtask = (
|
||||
harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
parent_task, parent_subtask = harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
input_uri = _upload_fixture_csv(harness.csv_client)
|
||||
|
||||
|
|
@ -230,10 +228,8 @@ def test_split_and_dispatch_publishes_one_message_per_child_with_matching_ids(
|
|||
harness: Harness,
|
||||
) -> None:
|
||||
# arrange
|
||||
parent_task, parent_subtask = (
|
||||
harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
parent_task, parent_subtask = harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
input_uri = _upload_fixture_csv(harness.csv_client)
|
||||
|
||||
|
|
@ -267,10 +263,8 @@ def test_split_and_dispatch_returns_child_ids_in_dispatch_order(
|
|||
harness: Harness,
|
||||
) -> None:
|
||||
# arrange
|
||||
parent_task, parent_subtask = (
|
||||
harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
parent_task, parent_subtask = harness.task_orchestrator.create_task_with_subtask(
|
||||
task_source="manual:postcode-splitter-int"
|
||||
)
|
||||
input_uri = _upload_fixture_csv(harness.csv_client)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@ from collections.abc import Iterator
|
|||
import pytest
|
||||
from moto import mock_aws
|
||||
|
||||
from domain.addresses.user_address import LandlordAssetList
|
||||
from domain.addresses.unsanitised_address import AddressList, UnsanitisedAddress
|
||||
from domain.postcode import Postcode
|
||||
from infrastructure.csv_s3_client import CsvS3Client
|
||||
from repositories.user_address.user_address_csv_s3_repository import (
|
||||
UserAddressCsvS3Repository,
|
||||
from repositories.unsanitised_address.unsanitised_address_list_csv_s3_repository import (
|
||||
UnsanitisedAddressListCsvS3Repository,
|
||||
)
|
||||
from tests.infrastructure import make_boto_client
|
||||
|
||||
|
|
@ -15,22 +15,22 @@ BUCKET = "user-address-bucket"
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def repo() -> Iterator[UserAddressCsvS3Repository]:
|
||||
def repo() -> Iterator[UnsanitisedAddressListCsvS3Repository]:
|
||||
with mock_aws():
|
||||
boto_client = make_boto_client("s3")
|
||||
boto_client.create_bucket(Bucket=BUCKET)
|
||||
csv_client = CsvS3Client(boto_client, BUCKET)
|
||||
yield UserAddressCsvS3Repository(csv_client, BUCKET)
|
||||
yield UnsanitisedAddressListCsvS3Repository(csv_client, BUCKET)
|
||||
|
||||
|
||||
def _upload_csv(
|
||||
repo: UserAddressCsvS3Repository, rows: list[dict[str, str]], key: str
|
||||
repo: UnsanitisedAddressListCsvS3Repository, rows: list[dict[str, str]], key: str
|
||||
) -> str:
|
||||
return repo._csv_client.save_rows(rows, key) # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
|
||||
def test_load_batch_parses_address_postcode_and_reference(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
rows = [
|
||||
|
|
@ -50,13 +50,13 @@ def test_load_batch_parses_address_postcode_and_reference(
|
|||
# assert
|
||||
assert len(addresses) == 1
|
||||
address = addresses[0]
|
||||
assert address.user_address == "1 High Street, Flat 2, Townville"
|
||||
assert address.address == "1 High Street, Flat 2, Townville"
|
||||
assert address.postcode == Postcode("SW1A1AA")
|
||||
assert address.internal_reference == "REF-001"
|
||||
assert address.org_reference == "REF-001"
|
||||
|
||||
|
||||
def test_load_batch_uses_only_address_1_when_others_missing(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
rows = [
|
||||
|
|
@ -75,13 +75,13 @@ def test_load_batch_uses_only_address_1_when_others_missing(
|
|||
|
||||
# assert
|
||||
assert len(addresses) == 1
|
||||
assert addresses[0].user_address == "10 Cardiff Road"
|
||||
assert addresses[0].address == "10 Cardiff Road"
|
||||
assert addresses[0].postcode == Postcode("CF101AA")
|
||||
assert addresses[0].internal_reference == "REF-002"
|
||||
assert addresses[0].org_reference == "REF-002"
|
||||
|
||||
|
||||
def test_load_batch_handles_missing_internal_reference(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
rows = [
|
||||
|
|
@ -100,16 +100,16 @@ def test_load_batch_handles_missing_internal_reference(
|
|||
|
||||
# assert
|
||||
assert len(addresses) == 1
|
||||
assert addresses[0].user_address == "5 Park Lane"
|
||||
assert addresses[0].address == "5 Park Lane"
|
||||
assert addresses[0].postcode == Postcode("M11AA")
|
||||
assert addresses[0].internal_reference is None
|
||||
assert addresses[0].org_reference is None
|
||||
|
||||
|
||||
def test_load_batch_captures_full_source_row(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# A raw EPC-export-shaped row: the splitter must preserve every column,
|
||||
# not just the ones it parses into UserAddress fields.
|
||||
# not just the ones it parses into UnsanitisedAddress fields.
|
||||
# arrange
|
||||
row = {
|
||||
"Asset Reference": "511",
|
||||
|
|
@ -124,11 +124,11 @@ def test_load_batch_captures_full_source_row(
|
|||
addresses = repo.load_batch(uri)
|
||||
|
||||
# assert
|
||||
assert addresses[0].landlord_additional_info == row
|
||||
assert addresses[0].additional_info == row
|
||||
|
||||
|
||||
def test_load_batch_raises_when_postcode_column_absent(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
rows = [{"Address 1": "1 High Street", "Property Type": "Flat"}]
|
||||
|
|
@ -140,7 +140,7 @@ def test_load_batch_raises_when_postcode_column_absent(
|
|||
|
||||
|
||||
def test_save_batch_passes_through_all_columns_and_appends_postcode_clean(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
row = {
|
||||
|
|
@ -169,19 +169,21 @@ def test_save_batch_passes_through_all_columns_and_appends_postcode_clean(
|
|||
|
||||
|
||||
def test_save_batch_returns_uri_under_path_prefix(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
addresses = [
|
||||
LandlordAssetList(
|
||||
user_address="1 High Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
landlord_additional_info={
|
||||
"Address 1": "1 High Street",
|
||||
"postcode": "SW1A 1AA",
|
||||
},
|
||||
),
|
||||
]
|
||||
addresses = AddressList(
|
||||
[
|
||||
UnsanitisedAddress(
|
||||
address="1 High Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
additional_info={
|
||||
"Address 1": "1 High Street",
|
||||
"postcode": "SW1A 1AA",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# act
|
||||
uri = repo.save_batch(addresses, "tasks/abc/batches")
|
||||
|
|
@ -192,7 +194,7 @@ def test_save_batch_returns_uri_under_path_prefix(
|
|||
|
||||
|
||||
def test_save_then_reload_round_trip_preserves_columns(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
rows = [
|
||||
|
|
@ -225,19 +227,21 @@ def test_save_then_reload_round_trip_preserves_columns(
|
|||
|
||||
|
||||
def test_save_batch_uses_unique_filename_per_call(
|
||||
repo: UserAddressCsvS3Repository,
|
||||
repo: UnsanitisedAddressListCsvS3Repository,
|
||||
) -> None:
|
||||
# arrange
|
||||
addresses = [
|
||||
LandlordAssetList(
|
||||
user_address="1 High Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
landlord_additional_info={
|
||||
"Address 1": "1 High Street",
|
||||
"postcode": "SW1A 1AA",
|
||||
},
|
||||
),
|
||||
]
|
||||
addresses = AddressList(
|
||||
[
|
||||
UnsanitisedAddress(
|
||||
address="1 High Street",
|
||||
postcode=Postcode("SW1A 1AA"),
|
||||
additional_info={
|
||||
"Address 1": "1 High Street",
|
||||
"postcode": "SW1A 1AA",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# act
|
||||
uri_1 = repo.save_batch(addresses, "tasks/uniqueness")
|
||||
Loading…
Add table
Reference in a new issue