diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index e10fbec7b..3d581293f 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -1,6 +1,6 @@ from collections import defaultdict import os -from typing import Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional from datetime import datetime import requests @@ -29,6 +29,12 @@ class DownloadedFiles(NamedTuple): other: List[DownloadedFile] +class RdSapSummary(NamedTuple): + pre_sap_rating: Optional[float] + pre_current_co2_emissions: Optional[float] + pre_current_energy_consumption: Optional[float] + + class UnauthorizedError(Exception): pass @@ -126,6 +132,23 @@ class PashubClient: ) return None + def get_rdsap_summary_by_job_id(self, job_id: str) -> RdSapSummary: + logger.info(f"Getting RdSAP Summary for job ID {job_id}") + url = f"{self.base}/jobs/{job_id}/rdSapSummary" + + r = self.session.get(url) + if r.status_code == 401: + raise UnauthorizedError("Token expired or invalid") + + r.raise_for_status() + + data: Dict[str, Any] = r.json() + return RdSapSummary( + pre_sap_rating=data.get("preSapRating"), + pre_current_co2_emissions=data.get("preCurrentCo2Emissions"), + pre_current_energy_consumption=data.get("preCurrentEnergyConsumption"), + ) + def _group_into_core_and_other_files( self, files: List[EvidenceFileData], diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 87caa89b8..18caab333 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -1,6 +1,7 @@ import os +from dataclasses import replace from datetime import datetime, timezone -from typing import Callable, List, NamedTuple, Optional, cast +from typing import Any, Callable, Dict, List, NamedTuple, Optional, cast from backend.app.db.connection import db_session from infrastructure.postgres.uploaded_file_table import ( @@ -15,11 +16,13 @@ from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, UnauthorizedError, ) from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) +from datatypes.epc.domain.epc import Epc from datatypes.epc.domain.epc_property_data import EpcPropertyData from utils.logger import setup_logger from utils.s3 import upload_file_to_s3 @@ -28,6 +31,31 @@ from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient logger = setup_logger() +def _overlay_lodged_performance( + epc_data: EpcPropertyData, summary: RdSapSummary +) -> EpcPropertyData: + """Fill the as-surveyed current-performance block PasHub carries but the + Site Notes PDF does not. Best-effort per field: values PasHub returns as + null are left empty. The EPC Band is derived from the SAP score via + `Epc.from_sap_score` rather than PasHub's opaque band code.""" + updates: Dict[str, Any] = {} + + if summary.pre_sap_rating is not None: + score = int(summary.pre_sap_rating) + updates["energy_rating_current"] = score + updates["current_energy_efficiency_band"] = Epc.from_sap_score(score) + + if summary.pre_current_co2_emissions is not None: + updates["co2_emissions_current"] = summary.pre_current_co2_emissions + + if summary.pre_current_energy_consumption is not None: + updates["energy_consumption_current"] = int( + summary.pre_current_energy_consumption + ) + + return replace(epc_data, **updates) + + class _FileUploadRecord(NamedTuple): file_path: str file_type: Optional[str] @@ -91,6 +119,7 @@ class PashubService: job_id, include_other=request.get_other_files ) + upload_records: List[_FileUploadRecord] = [] if uprn or hubspot_deal_id: logger.info("Uploading files to s3") file_source = ( @@ -101,7 +130,6 @@ class PashubService: upload_records = self._upload_to_s3_and_update_db( downloaded.core, uprn, hubspot_deal_id, file_source ) - self._save_site_notes(upload_records) if downloaded.other: self._upload_to_s3_and_update_db( @@ -125,6 +153,11 @@ class PashubService: property_folder_path = f"{request.sharepoint_link}/{match}" self._upload_to_sharepoint(property_folder_path, downloaded.core + downloaded.other) + # Ordered after S3 + SharePoint so a loud RdSAP Summary failure cannot + # cost us file distribution: the Property refuses to persist with an + # incomplete Lodged Performance, but the evidence is already delivered. + self._save_site_notes(upload_records, active_client, job_id) + for df in downloaded.core + downloaded.other: try: os.remove(df.file_path) @@ -190,17 +223,44 @@ class PashubService: return upload_records - def _save_site_notes(self, upload_records: List[_FileUploadRecord]) -> None: + def _fetch_rdsap_summary( + self, active_client: PashubClient, job_id: str + ) -> RdSapSummary: + try: + return active_client.get_rdsap_summary_by_job_id(job_id) + except UnauthorizedError: + if active_client is not self._pashub_client: + raise + logger.info( + f"PasHub credentials unauthorized for RdSAP Summary on job {job_id}; " + "retrying with CoordinationHub credentials" + ) + return self._get_coordination_client().get_rdsap_summary_by_job_id(job_id) + + def _save_site_notes( + self, + upload_records: List[_FileUploadRecord], + active_client: PashubClient, + job_id: str, + ) -> None: for record in upload_records: if ( record.file_type is None or FileTypeEnum(record.file_type) != FileTypeEnum.RD_SAP_SITE_NOTE ): continue + + # The RdSAP Summary carries the as-surveyed Lodged Performance the + # Site Notes PDF lacks. Fetch it outside the parse try/except so a + # missing summary fails loudly rather than persisting a Property with + # an incomplete Lodged Performance. + summary: RdSapSummary = self._fetch_rdsap_summary(active_client, job_id) + try: epc_data: EpcPropertyData = parse_site_notes_pdf( record.file_path, uprn=record.uprn ) + epc_data = _overlay_lodged_performance(epc_data, summary) with db_session() as session: save_epc_property_data( session, epc_data, uploaded_file_id=record.uploaded_file_id diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/backend/pashub_fetcher/tests/test_pashub_client.py index 214a14a6a..67b2ebac5 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/backend/pashub_fetcher/tests/test_pashub_client.py @@ -1,14 +1,18 @@ # pyright: reportPrivateUsage=false -from typing import Optional -from unittest.mock import patch +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest +import requests from backend.pashub_fetcher.core_files import CoreFiles from backend.pashub_fetcher.evidence_file_data import EvidenceFileData from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata from backend.pashub_fetcher.pashub_client import ( - DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, + UnauthorizedError, ) @@ -278,3 +282,67 @@ def test_get_evidence_files_by_job_id_downloads_other_files_when_include_other_t # Assert assert [df.file_path for df in result.core] == ["/tmp/SiteNote_001.pdf"] assert [df.file_path for df in result.other] == ["/tmp/unknown_doc.pdf"] + + +# --------------------------------------------------------------------------- +# get_rdsap_summary_by_job_id +# --------------------------------------------------------------------------- + + +def make_response( + status_code: int = 200, + payload: Optional[dict[str, Any]] = None, + raise_on_status: Optional[Exception] = None, +) -> MagicMock: + response = MagicMock() + response.status_code = status_code + response.json.return_value = payload if payload is not None else {} + if raise_on_status is not None: + response.raise_for_status.side_effect = raise_on_status + return response + + +def test_get_rdsap_summary_maps_pre_performance_fields() -> None: + # Arrange + client = make_client() + payload = { + "preSapRating": 78.00, + "preSapBand": 50, + "preCurrentEnergyConsumption": 117.00, + "preCurrentCo2Emissions": 1.70, + } + + # Act + client.session.get = MagicMock(return_value=make_response(payload=payload)) + result = client.get_rdsap_summary_by_job_id("job-1") + + # Assert + assert result == RdSapSummary( + pre_sap_rating=78.00, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.00, + ) + + +def test_get_rdsap_summary_raises_unauthorized_on_401() -> None: + # Arrange + client = make_client() + + # Act / Assert + client.session.get = MagicMock(return_value=make_response(status_code=401)) + with pytest.raises(UnauthorizedError): + client.get_rdsap_summary_by_job_id("job-1") + + +def test_get_rdsap_summary_raises_on_http_error() -> None: + # Arrange + client = make_client() + + # Act / Assert — a 404/error must raise loudly, not resolve to empty values + client.session.get = MagicMock( + return_value=make_response( + status_code=404, raise_on_status=requests.HTTPError("not found") + ) + ) + with pytest.raises(requests.HTTPError): + client.get_rdsap_summary_by_job_id("job-1") diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index 09635c924..1acb583d3 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -1,5 +1,5 @@ import pytest -from datetime import datetime +from datetime import date, datetime from typing import Any, Callable, Optional from unittest.mock import MagicMock, call, patch @@ -9,16 +9,74 @@ from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, UnauthorizedError, ) from backend.pashub_fetcher.pashub_service import PashubService from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) +from datatypes.epc.domain.epc import Epc +from datatypes.epc.domain.epc_property_data import ( + EpcPropertyData, + SapEnergySource, + SapHeating, +) from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient FAKE_JOB_LINK = "https://pashub.net/jobs/job-id-123/details" +RD_SAP_SITE_NOTE_PATH = "/tmp/RdSAP_SiteNote_001.pdf" + + +def make_epc_property_data() -> EpcPropertyData: + """A minimal, real EpcPropertyData whose as-surveyed performance block is + empty — mirroring a freshly parsed Site Notes PDF the overlay fills in.""" + return EpcPropertyData( + dwelling_type="House", + inspection_date=date(2024, 1, 1), + tenure="1", + transaction_type="1", + address_line_1="1 Test Street", + postcode="AB1 2CD", + post_town="Testville", + roofs=[], + walls=[], + floors=[], + main_heating=[], + door_count=0, + sap_heating=SapHeating( + instantaneous_wwhrs=None, + main_heating_details=[], + has_fixed_air_conditioning=False, + ), + sap_windows=[], + sap_energy_source=SapEnergySource( + gas_connection_available=False, + meter_type="Single", + pv_battery_count=0, + wind_turbines_count=0, + gas_smart_meter_present=False, + is_dwelling_export_capable=False, + wind_turbines_terrain_type="Suburban", + electricity_smart_meter_present=False, + ), + sap_building_parts=[], + solar_water_heating=False, + has_hot_water_cylinder=False, + has_fixed_air_conditioning=False, + wet_rooms_count=0, + extensions_count=0, + heated_rooms_count=0, + open_chimneys_count=0, + habitable_rooms_count=0, + insulated_door_count=0, + cfl_fixed_lighting_bulbs_count=0, + led_fixed_lighting_bulbs_count=0, + incandescent_fixed_lighting_bulbs_count=0, + total_floor_area_m2=80.0, + ) + def make_request( pashub_link: str = FAKE_JOB_LINK, @@ -232,13 +290,15 @@ def test_run_uses_hubspot_deal_id_path_when_no_uprn() -> None: def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: - mock_client = MagicMock(spec=PashubClient) - mock_client.get_uprn_by_job_id.return_value = None - mock_client.get_evidence_files_by_job_id.return_value = make_downloaded( - core=["/tmp/RdSAP_SiteNote_001.pdf"] + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=None, + pre_current_co2_emissions=None, + pre_current_energy_consumption=None, + ) ) - fake_epc_data = MagicMock() + parsed = make_epc_property_data() fake_session = MagicMock() fake_uploaded_file_id = 99 @@ -248,7 +308,7 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), patch( "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", - return_value=fake_epc_data, + return_value=parsed, ) as mock_parse, patch( "backend.pashub_fetcher.pashub_service.save_epc_property_data" @@ -262,9 +322,9 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345")) - mock_parse.assert_called_once_with("/tmp/RdSAP_SiteNote_001.pdf") + mock_parse.assert_called_once_with(RD_SAP_SITE_NOTE_PATH, uprn=12345) mock_save.assert_called_once_with( - fake_session, fake_epc_data, uploaded_file_id=fake_uploaded_file_id + fake_session, parsed, uploaded_file_id=fake_uploaded_file_id ) @@ -670,3 +730,200 @@ def test_run_warns_and_continues_when_site_notes_parsing_fails() -> None: assert result == ["/tmp/RdSAP_SiteNote_001.pdf"] mock_logger.warning.assert_called() mock_save.assert_not_called() + + +# --------------------------------------------------------------------------- +# run(): RdSAP Summary overlaid onto Site-Notes Lodged Performance +# --------------------------------------------------------------------------- + + +def _site_note_client( + summary: Any = None, + core: Optional[list[str]] = None, +) -> MagicMock: + mock_client = MagicMock(spec=PashubClient) + mock_client.get_uprn_by_job_id.return_value = None + mock_client.get_evidence_files_by_job_id.return_value = make_downloaded( + core=core if core is not None else [RD_SAP_SITE_NOTE_PATH] + ) + mock_client.get_rdsap_summary_by_job_id.return_value = summary + return mock_client + + +def test_run_overlays_full_lodged_performance_onto_saved_site_notes() -> None: + # Arrange + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.0, + ) + ) + parsed = make_epc_property_data() + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 + assert saved.current_energy_efficiency_band == Epc.C + assert saved.co2_emissions_current == 1.70 + assert saved.energy_consumption_current == 117 + + +def test_run_overlays_only_present_fields_when_summary_partial() -> None: + # Arrange — only the SAP rating comes back; CO2 and PEUI are null + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=None, + pre_current_energy_consumption=None, + ) + ) + parsed = make_epc_property_data() + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 + assert saved.current_energy_efficiency_band == Epc.C + assert saved.co2_emissions_current is None + assert saved.energy_consumption_current is None + + +def test_run_raises_and_skips_persistence_when_summary_fetch_fails() -> None: + # Arrange — the RdSAP Site Note is present but its summary cannot be fetched + mock_client = _site_note_client() + mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down") + service = make_service(pashub_client=mock_client) + + # Act / Assert + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + with pytest.raises(RuntimeError): + service.run(make_request(uprn="12345")) + + mock_save.assert_not_called() + mock_s3.assert_called() # S3 upload happened before the loud site-note save + + +def test_run_does_not_fetch_summary_for_evidence_only_job() -> None: + # Arrange — a plain Site Note (not an RdSAP Site Note) carries no summary + mock_client = _site_note_client(core=["/tmp/SiteNote_001.pdf"]) + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + mock_client.get_rdsap_summary_by_job_id.assert_not_called() + + +def test_run_fetches_summary_from_coordination_client_on_primary_auth_failure() -> None: + # Arrange — primary credentials 401 on the summary; coordination succeeds + pas_client = _site_note_client() + pas_client.get_rdsap_summary_by_job_id.side_effect = UnauthorizedError() + + coord_client = MagicMock(spec=PashubClient) + coord_client.get_rdsap_summary_by_job_id.return_value = RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.0, + ) + factory = MagicMock(return_value=coord_client) + + parsed = make_epc_property_data() + service = make_service( + pashub_client=pas_client, coordination_client_factory=factory + ) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + coord_client.get_rdsap_summary_by_job_id.assert_called_once() + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 + + +def test_run_uploads_to_sharepoint_even_when_summary_fetch_fails() -> None: + # Arrange — file distribution must survive a loud site-note save failure + mock_client = _site_note_client() + mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down") + + mock_sharepoint = MagicMock(spec=DomnaSharepointClient) + mock_sharepoint.get_folders_in_path.return_value = { + "value": [{"name": "123 Main St"}] + } + service = make_service(pashub_client=mock_client, sharepoint_client=mock_sharepoint) + + # Act / Assert + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + with pytest.raises(RuntimeError): + service.run( + make_request( + uprn="12345", + sharepoint_link="Retrofit/Properties", + address="123 Main St | deal", + ) + ) + + mock_sharepoint.upload_file.assert_any_call( + RD_SAP_SITE_NOTE_PATH, "Retrofit/Properties/123 Main St", "RdSAP_SiteNote_001.pdf" + )