diff --git a/backend/documents_parser/parser.py b/backend/documents_parser/parser.py index cff21e0ef..57426ab87 100644 --- a/backend/documents_parser/parser.py +++ b/backend/documents_parser/parser.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from datatypes.epc.domain.epc_property_data import EpcPropertyData from datatypes.epc.domain.mapper import EpcPropertyDataMapper @@ -8,13 +8,15 @@ from backend.documents_parser.extractor import PasHubRdSapSiteNotesExtractor from backend.documents_parser.pdf import pdf_to_pages, pdf_to_text_list -def parse_site_notes_pdf(file_path: str) -> EpcPropertyData: +def parse_site_notes_pdf( + file_path: str, uprn: Optional[int] = None +) -> EpcPropertyData: with open(file_path, "rb") as f: pdf_bytes = f.read() pages = pdf_to_pages(pdf_bytes) if "Elmhurst Energy Systems" in "\n".join(pages): return _parse_elmhurst(pages) - return _parse_pashub(pdf_bytes) + return _parse_pashub(pdf_bytes, uprn=uprn) def _parse_elmhurst(pages: List[str]) -> EpcPropertyData: @@ -22,7 +24,9 @@ def _parse_elmhurst(pages: List[str]) -> EpcPropertyData: return EpcPropertyDataMapper.from_elmhurst_site_notes(site_notes) -def _parse_pashub(pdf_bytes: bytes) -> EpcPropertyData: +def _parse_pashub( + pdf_bytes: bytes, uprn: Optional[int] = None +) -> EpcPropertyData: tokens = pdf_to_text_list(pdf_bytes) site_notes = PasHubRdSapSiteNotesExtractor(tokens).extract() - return EpcPropertyDataMapper.from_site_notes(site_notes) + return EpcPropertyDataMapper.from_site_notes(site_notes, uprn=uprn) diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 64c2f6aed..87caa89b8 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -32,6 +32,7 @@ class _FileUploadRecord(NamedTuple): file_path: str file_type: Optional[str] uploaded_file_id: int + uprn: Optional[int] class PashubService: @@ -149,6 +150,10 @@ class PashubService: else f"documents/hubspot_deal_id/{hubspot_deal_id}" ) + # Coerce the service-held UPRN string to an int in exactly one place so + # the site-notes row's UPRN cannot diverge from the UploadedFile row's. + uprn_int: Optional[int] = int(uprn) if uprn else None + file_paths: List[str] = [] uploaded_files: List[UploadedFile] = [] @@ -162,7 +167,7 @@ class PashubService: s3_file_bucket=self._s3_bucket, s3_file_key=file_key, s3_upload_timestamp=datetime.now(timezone.utc), - uprn=int(uprn) if uprn else None, + uprn=uprn_int, hubspot_deal_id=hubspot_deal_id, file_source=file_source.value, file_type=get_file_type_string(filename, df.evidence_category) or default_file_type, @@ -178,6 +183,7 @@ class PashubService: file_path=fp, file_type=cast(Optional[str], uf.file_type), uploaded_file_id=cast(int, uf.id), + uprn=uprn_int, ) for fp, uf in zip(file_paths, uploaded_files) ] @@ -192,7 +198,9 @@ class PashubService: ): continue try: - epc_data: EpcPropertyData = parse_site_notes_pdf(record.file_path) + epc_data: EpcPropertyData = parse_site_notes_pdf( + record.file_path, uprn=record.uprn + ) with db_session() as session: save_epc_property_data( session, epc_data, uploaded_file_id=record.uploaded_file_id diff --git a/datatypes/epc/domain/mapper.py b/datatypes/epc/domain/mapper.py index 83e232245..ad9883bc3 100644 --- a/datatypes/epc/domain/mapper.py +++ b/datatypes/epc/domain/mapper.py @@ -280,7 +280,9 @@ def _map_schema_21_pv( class EpcPropertyDataMapper: @staticmethod - def from_site_notes(survey: PasHubRdSapSiteNotes) -> EpcPropertyData: + def from_site_notes( + survey: PasHubRdSapSiteNotes, uprn: Optional[int] = None + ) -> EpcPropertyData: general = survey.general metadata = survey.inspection_metadata address_parts = [p.strip() for p in metadata.property_address.split(", ")] @@ -336,6 +338,7 @@ class EpcPropertyDataMapper: ) # TODO: verify that is the correct approach return EpcPropertyData( + uprn=uprn, dwelling_type=f"{general.detachment_type} {general.property_type.lower()}", inspection_date=general.inspection_date, tenure=general.tenure, diff --git a/datatypes/epc/domain/tests/test_from_site_notes.py b/datatypes/epc/domain/tests/test_from_site_notes.py index 5cb29b698..bde715237 100644 --- a/datatypes/epc/domain/tests/test_from_site_notes.py +++ b/datatypes/epc/domain/tests/test_from_site_notes.py @@ -723,6 +723,29 @@ class TestUnmeasurableWallThickness: assert result.sap_building_parts[0].wall_thickness_mm is None +class TestFromSiteNotesUprnInjection: + """Site notes never carry a UPRN natively, but the PasHub path can inject a + UPRN the service has already resolved. A supplied UPRN must land on the + resulting EpcPropertyData; the default (none supplied) must yield no UPRN. + """ + + @pytest.fixture + def survey(self) -> PasHubRdSapSiteNotes: + return from_dict( + PasHubRdSapSiteNotes, load("pashub_rdsap_site_notes_example1.json") + ) + + def test_supplied_uprn_lands_on_result( + self, survey: PasHubRdSapSiteNotes + ) -> None: + result = EpcPropertyDataMapper.from_site_notes(survey, uprn=100010344955) + assert result.uprn == 100010344955 + + def test_default_yields_no_uprn(self, survey: PasHubRdSapSiteNotes) -> None: + result = EpcPropertyDataMapper.from_site_notes(survey) + assert result.uprn is None + + class TestElmhurstSecondaryFuelFromSapCode: """`_elmhurst_secondary_fuel_from_sap_code` must RAISE on an unmapped fuel-fired Table 4a Category-10 room-heater code rather than return