From 85482cdf9effa0294ca1671444c60bbfd1e2a015 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:40:18 +0000 Subject: [PATCH 01/12] =?UTF-8?q?Map=20PasHub=20RdSAP=20Summary=20response?= =?UTF-8?q?=20to=20as-surveyed=20performance=20values=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_client.py | 9 ++++ .../tests/test_pashub_client.py | 49 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index e10fbec7b..a3fa07714 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -29,6 +29,12 @@ class DownloadedFiles(NamedTuple): other: List[DownloadedFile] +class RdSapSummary(NamedTuple): + pre_sap_rating: Optional[float] + pre_current_co2_emissions: Optional[float] + pre_current_energy_consumption: Optional[float] + + class UnauthorizedError(Exception): pass @@ -126,6 +132,9 @@ class PashubClient: ) return None + def get_rdsap_summary_by_job_id(self, job_id: str) -> RdSapSummary: + raise NotImplementedError + def _group_into_core_and_other_files( self, files: List[EvidenceFileData], diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/backend/pashub_fetcher/tests/test_pashub_client.py index 214a14a6a..4f6139530 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/backend/pashub_fetcher/tests/test_pashub_client.py @@ -1,6 +1,9 @@ # pyright: reportPrivateUsage=false -from typing import Optional -from unittest.mock import patch +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest +import requests from backend.pashub_fetcher.core_files import CoreFiles from backend.pashub_fetcher.evidence_file_data import EvidenceFileData @@ -9,6 +12,8 @@ from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, + UnauthorizedError, ) @@ -278,3 +283,43 @@ def test_get_evidence_files_by_job_id_downloads_other_files_when_include_other_t # Assert assert [df.file_path for df in result.core] == ["/tmp/SiteNote_001.pdf"] assert [df.file_path for df in result.other] == ["/tmp/unknown_doc.pdf"] + + +# --------------------------------------------------------------------------- +# get_rdsap_summary_by_job_id +# --------------------------------------------------------------------------- + + +def make_response( + status_code: int = 200, + payload: Optional[dict[str, Any]] = None, + raise_on_status: Optional[Exception] = None, +) -> MagicMock: + response = MagicMock() + response.status_code = status_code + response.json.return_value = payload if payload is not None else {} + if raise_on_status is not None: + response.raise_for_status.side_effect = raise_on_status + return response + + +def test_get_rdsap_summary_maps_pre_performance_fields() -> None: + # Arrange + client = make_client() + payload = { + "preSapRating": 78.00, + "preSapBand": 50, + "preCurrentEnergyConsumption": 117.00, + "preCurrentCo2Emissions": 1.70, + } + + # Act + client.session.get = MagicMock(return_value=make_response(payload=payload)) + result = client.get_rdsap_summary_by_job_id("job-1") + + # Assert + assert result == RdSapSummary( + pre_sap_rating=78.00, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.00, + ) From af6c461c933079a8d61e688e7306c8824a95f1b0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:40:55 +0000 Subject: [PATCH 02/12] =?UTF-8?q?Map=20PasHub=20RdSAP=20Summary=20response?= =?UTF-8?q?=20to=20as-surveyed=20performance=20values=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_client.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index a3fa07714..2ca34367d 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -1,6 +1,6 @@ from collections import defaultdict import os -from typing import Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional from datetime import datetime import requests @@ -133,7 +133,17 @@ class PashubClient: return None def get_rdsap_summary_by_job_id(self, job_id: str) -> RdSapSummary: - raise NotImplementedError + logger.info(f"Getting RdSAP Summary for job ID {job_id}") + url = f"{self.base}/jobs/{job_id}/rdSapSummary" + + r = self.session.get(url) + + data: Dict[str, Any] = r.json() + return RdSapSummary( + pre_sap_rating=data.get("preSapRating"), + pre_current_co2_emissions=data.get("preCurrentCo2Emissions"), + pre_current_energy_consumption=data.get("preCurrentEnergyConsumption"), + ) def _group_into_core_and_other_files( self, From b596f32af1e0dacab286e04e2d7194f7d55e505a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:41:27 +0000 Subject: [PATCH 03/12] =?UTF-8?q?Raise=20UnauthorizedError=20when=20RdSAP?= =?UTF-8?q?=20Summary=20fetch=20returns=20401=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/tests/test_pashub_client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/backend/pashub_fetcher/tests/test_pashub_client.py index 4f6139530..1e6bbadb3 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/backend/pashub_fetcher/tests/test_pashub_client.py @@ -323,3 +323,13 @@ def test_get_rdsap_summary_maps_pre_performance_fields() -> None: pre_current_co2_emissions=1.70, pre_current_energy_consumption=117.00, ) + + +def test_get_rdsap_summary_raises_unauthorized_on_401() -> None: + # Arrange + client = make_client() + + # Act / Assert + client.session.get = MagicMock(return_value=make_response(status_code=401)) + with pytest.raises(UnauthorizedError): + client.get_rdsap_summary_by_job_id("job-1") From 83d92010de06bef235393788c8be003b40261882 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:41:43 +0000 Subject: [PATCH 04/12] =?UTF-8?q?Raise=20UnauthorizedError=20when=20RdSAP?= =?UTF-8?q?=20Summary=20fetch=20returns=20401=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index 2ca34367d..8ae84e617 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -137,6 +137,8 @@ class PashubClient: url = f"{self.base}/jobs/{job_id}/rdSapSummary" r = self.session.get(url) + if r.status_code == 401: + raise UnauthorizedError("Token expired or invalid") data: Dict[str, Any] = r.json() return RdSapSummary( From eff4f644265c38d8832f97ec6dcb815b2f31d7f7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:43:05 +0000 Subject: [PATCH 05/12] =?UTF-8?q?Raise=20loudly=20when=20RdSAP=20Summary?= =?UTF-8?q?=20fetch=20returns=20an=20HTTP=20error=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/tests/test_pashub_client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/backend/pashub_fetcher/tests/test_pashub_client.py index 1e6bbadb3..48731eea6 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/backend/pashub_fetcher/tests/test_pashub_client.py @@ -333,3 +333,17 @@ def test_get_rdsap_summary_raises_unauthorized_on_401() -> None: client.session.get = MagicMock(return_value=make_response(status_code=401)) with pytest.raises(UnauthorizedError): client.get_rdsap_summary_by_job_id("job-1") + + +def test_get_rdsap_summary_raises_on_http_error() -> None: + # Arrange + client = make_client() + + # Act / Assert — a 404/error must raise loudly, not resolve to empty values + client.session.get = MagicMock( + return_value=make_response( + status_code=404, raise_on_status=requests.HTTPError("not found") + ) + ) + with pytest.raises(requests.HTTPError): + client.get_rdsap_summary_by_job_id("job-1") From ded8cf1e46ba3647c1d5bb54ffe72492d82702a1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:43:27 +0000 Subject: [PATCH 06/12] =?UTF-8?q?Raise=20loudly=20when=20RdSAP=20Summary?= =?UTF-8?q?=20fetch=20returns=20an=20HTTP=20error=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index 8ae84e617..3d581293f 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -140,6 +140,8 @@ class PashubClient: if r.status_code == 401: raise UnauthorizedError("Token expired or invalid") + r.raise_for_status() + data: Dict[str, Any] = r.json() return RdSapSummary( pre_sap_rating=data.get("preSapRating"), From 43fac405950a5097f19120b4d0be8199222ab388 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:45:04 +0000 Subject: [PATCH 07/12] =?UTF-8?q?Overlay=20PasHub=20as-surveyed=20performa?= =?UTF-8?q?nce=20onto=20saved=20Site-Notes=20property=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_pashub_service.py | 113 +++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index 09635c924..4cec4ac67 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -1,5 +1,5 @@ import pytest -from datetime import datetime +from datetime import date, datetime from typing import Any, Callable, Optional from unittest.mock import MagicMock, call, patch @@ -9,16 +9,74 @@ from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, UnauthorizedError, ) from backend.pashub_fetcher.pashub_service import PashubService from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) +from datatypes.epc.domain.epc import Epc +from datatypes.epc.domain.epc_property_data import ( + EpcPropertyData, + SapEnergySource, + SapHeating, +) from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient FAKE_JOB_LINK = "https://pashub.net/jobs/job-id-123/details" +RD_SAP_SITE_NOTE_PATH = "/tmp/RdSAP_SiteNote_001.pdf" + + +def make_epc_property_data() -> EpcPropertyData: + """A minimal, real EpcPropertyData whose as-surveyed performance block is + empty — mirroring a freshly parsed Site Notes PDF the overlay fills in.""" + return EpcPropertyData( + dwelling_type="House", + inspection_date=date(2024, 1, 1), + tenure="1", + transaction_type="1", + address_line_1="1 Test Street", + postcode="AB1 2CD", + post_town="Testville", + roofs=[], + walls=[], + floors=[], + main_heating=[], + door_count=0, + sap_heating=SapHeating( + instantaneous_wwhrs=None, + main_heating_details=[], + has_fixed_air_conditioning=False, + ), + sap_windows=[], + sap_energy_source=SapEnergySource( + gas_connection_available=False, + meter_type="Single", + pv_battery_count=0, + wind_turbines_count=0, + gas_smart_meter_present=False, + is_dwelling_export_capable=False, + wind_turbines_terrain_type="Suburban", + electricity_smart_meter_present=False, + ), + sap_building_parts=[], + solar_water_heating=False, + has_hot_water_cylinder=False, + has_fixed_air_conditioning=False, + wet_rooms_count=0, + extensions_count=0, + heated_rooms_count=0, + open_chimneys_count=0, + habitable_rooms_count=0, + insulated_door_count=0, + cfl_fixed_lighting_bulbs_count=0, + led_fixed_lighting_bulbs_count=0, + incandescent_fixed_lighting_bulbs_count=0, + total_floor_area_m2=80.0, + ) + def make_request( pashub_link: str = FAKE_JOB_LINK, @@ -670,3 +728,56 @@ def test_run_warns_and_continues_when_site_notes_parsing_fails() -> None: assert result == ["/tmp/RdSAP_SiteNote_001.pdf"] mock_logger.warning.assert_called() mock_save.assert_not_called() + + +# --------------------------------------------------------------------------- +# run(): RdSAP Summary overlaid onto Site-Notes Lodged Performance +# --------------------------------------------------------------------------- + + +def _site_note_client( + summary: Any = None, + core: Optional[list[str]] = None, +) -> MagicMock: + mock_client = MagicMock(spec=PashubClient) + mock_client.get_uprn_by_job_id.return_value = None + mock_client.get_evidence_files_by_job_id.return_value = make_downloaded( + core=core if core is not None else [RD_SAP_SITE_NOTE_PATH] + ) + mock_client.get_rdsap_summary_by_job_id.return_value = summary + return mock_client + + +def test_run_overlays_full_lodged_performance_onto_saved_site_notes() -> None: + # Arrange + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.0, + ) + ) + parsed = make_epc_property_data() + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 + assert saved.current_energy_efficiency_band == Epc.C + assert saved.co2_emissions_current == 1.70 + assert saved.energy_consumption_current == 117 From 377c030c77b5ab061678213192444ff8dd1c9c07 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:47:49 +0000 Subject: [PATCH 08/12] =?UTF-8?q?Overlay=20PasHub=20as-surveyed=20performa?= =?UTF-8?q?nce=20onto=20saved=20Site-Notes=20property=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_service.py | 61 ++++++++++++++++++- .../tests/test_pashub_service.py | 18 +++--- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 87caa89b8..02ca24003 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -1,6 +1,7 @@ import os +from dataclasses import replace from datetime import datetime, timezone -from typing import Callable, List, NamedTuple, Optional, cast +from typing import Any, Callable, Dict, List, NamedTuple, Optional, cast from backend.app.db.connection import db_session from infrastructure.postgres.uploaded_file_table import ( @@ -15,11 +16,13 @@ from backend.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, + RdSapSummary, UnauthorizedError, ) from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) +from datatypes.epc.domain.epc import Epc from datatypes.epc.domain.epc_property_data import EpcPropertyData from utils.logger import setup_logger from utils.s3 import upload_file_to_s3 @@ -28,6 +31,31 @@ from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient logger = setup_logger() +def _overlay_lodged_performance( + epc_data: EpcPropertyData, summary: RdSapSummary +) -> EpcPropertyData: + """Fill the as-surveyed current-performance block PasHub carries but the + Site Notes PDF does not. Best-effort per field: values PasHub returns as + null are left empty. The EPC Band is derived from the SAP score via + `Epc.from_sap_score` rather than PasHub's opaque band code.""" + updates: Dict[str, Any] = {} + + if summary.pre_sap_rating is not None: + score = int(summary.pre_sap_rating) + updates["energy_rating_current"] = score + updates["current_energy_efficiency_band"] = Epc.from_sap_score(score) + + if summary.pre_current_co2_emissions is not None: + updates["co2_emissions_current"] = summary.pre_current_co2_emissions + + if summary.pre_current_energy_consumption is not None: + updates["energy_consumption_current"] = int( + summary.pre_current_energy_consumption + ) + + return replace(epc_data, **updates) + + class _FileUploadRecord(NamedTuple): file_path: str file_type: Optional[str] @@ -101,7 +129,7 @@ class PashubService: upload_records = self._upload_to_s3_and_update_db( downloaded.core, uprn, hubspot_deal_id, file_source ) - self._save_site_notes(upload_records) + self._save_site_notes(upload_records, active_client, job_id) if downloaded.other: self._upload_to_s3_and_update_db( @@ -190,17 +218,44 @@ class PashubService: return upload_records - def _save_site_notes(self, upload_records: List[_FileUploadRecord]) -> None: + def _fetch_rdsap_summary( + self, active_client: PashubClient, job_id: str + ) -> RdSapSummary: + try: + return active_client.get_rdsap_summary_by_job_id(job_id) + except UnauthorizedError: + if active_client is not self._pashub_client: + raise + logger.info( + f"PasHub credentials unauthorized for RdSAP Summary on job {job_id}; " + "retrying with CoordinationHub credentials" + ) + return self._get_coordination_client().get_rdsap_summary_by_job_id(job_id) + + def _save_site_notes( + self, + upload_records: List[_FileUploadRecord], + active_client: PashubClient, + job_id: str, + ) -> None: for record in upload_records: if ( record.file_type is None or FileTypeEnum(record.file_type) != FileTypeEnum.RD_SAP_SITE_NOTE ): continue + + # The RdSAP Summary carries the as-surveyed Lodged Performance the + # Site Notes PDF lacks. Fetch it outside the parse try/except so a + # missing summary fails loudly rather than persisting a Property with + # an incomplete Lodged Performance. + summary: RdSapSummary = self._fetch_rdsap_summary(active_client, job_id) + try: epc_data: EpcPropertyData = parse_site_notes_pdf( record.file_path, uprn=record.uprn ) + epc_data = _overlay_lodged_performance(epc_data, summary) with db_session() as session: save_epc_property_data( session, epc_data, uploaded_file_id=record.uploaded_file_id diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index 4cec4ac67..d92c10979 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -290,13 +290,15 @@ def test_run_uses_hubspot_deal_id_path_when_no_uprn() -> None: def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: - mock_client = MagicMock(spec=PashubClient) - mock_client.get_uprn_by_job_id.return_value = None - mock_client.get_evidence_files_by_job_id.return_value = make_downloaded( - core=["/tmp/RdSAP_SiteNote_001.pdf"] + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=None, + pre_current_co2_emissions=None, + pre_current_energy_consumption=None, + ) ) - fake_epc_data = MagicMock() + parsed = make_epc_property_data() fake_session = MagicMock() fake_uploaded_file_id = 99 @@ -306,7 +308,7 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), patch( "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", - return_value=fake_epc_data, + return_value=parsed, ) as mock_parse, patch( "backend.pashub_fetcher.pashub_service.save_epc_property_data" @@ -320,9 +322,9 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345")) - mock_parse.assert_called_once_with("/tmp/RdSAP_SiteNote_001.pdf") + mock_parse.assert_called_once_with(RD_SAP_SITE_NOTE_PATH, uprn=12345) mock_save.assert_called_once_with( - fake_session, fake_epc_data, uploaded_file_id=fake_uploaded_file_id + fake_session, parsed, uploaded_file_id=fake_uploaded_file_id ) From 62c0151469948e2fb3c8f062612a1520ed089fbc Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:49:02 +0000 Subject: [PATCH 09/12] =?UTF-8?q?Fill=20partial=20summaries=20best-effort,?= =?UTF-8?q?=20fail=20loudly,=20skip=20evidence-only=20jobs,=20and=20retry?= =?UTF-8?q?=20on=20Coordination=20Hub=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_pashub_service.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index d92c10979..aa8b6e2ce 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -783,3 +783,116 @@ def test_run_overlays_full_lodged_performance_onto_saved_site_notes() -> None: assert saved.current_energy_efficiency_band == Epc.C assert saved.co2_emissions_current == 1.70 assert saved.energy_consumption_current == 117 + + +def test_run_overlays_only_present_fields_when_summary_partial() -> None: + # Arrange — only the SAP rating comes back; CO2 and PEUI are null + mock_client = _site_note_client( + summary=RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=None, + pre_current_energy_consumption=None, + ) + ) + parsed = make_epc_property_data() + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 + assert saved.current_energy_efficiency_band == Epc.C + assert saved.co2_emissions_current is None + assert saved.energy_consumption_current is None + + +def test_run_raises_and_skips_persistence_when_summary_fetch_fails() -> None: + # Arrange — the RdSAP Site Note is present but its summary cannot be fetched + mock_client = _site_note_client() + mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down") + service = make_service(pashub_client=mock_client) + + # Act / Assert + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + with pytest.raises(RuntimeError): + service.run(make_request(uprn="12345")) + + mock_save.assert_not_called() + mock_s3.assert_called() # S3 upload happened before the loud site-note save + + +def test_run_does_not_fetch_summary_for_evidence_only_job() -> None: + # Arrange — a plain Site Note (not an RdSAP Site Note) carries no summary + mock_client = _site_note_client(core=["/tmp/SiteNote_001.pdf"]) + service = make_service(pashub_client=mock_client) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + mock_client.get_rdsap_summary_by_job_id.assert_not_called() + + +def test_run_fetches_summary_from_coordination_client_on_primary_auth_failure() -> None: + # Arrange — primary credentials 401 on the summary; coordination succeeds + pas_client = _site_note_client() + pas_client.get_rdsap_summary_by_job_id.side_effect = UnauthorizedError() + + coord_client = MagicMock(spec=PashubClient) + coord_client.get_rdsap_summary_by_job_id.return_value = RdSapSummary( + pre_sap_rating=78.0, + pre_current_co2_emissions=1.70, + pre_current_energy_consumption=117.0, + ) + factory = MagicMock(return_value=coord_client) + + parsed = make_epc_property_data() + service = make_service( + pashub_client=pas_client, coordination_client_factory=factory + ) + + # Act + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch( + "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + return_value=parsed, + ), + patch( + "backend.pashub_fetcher.pashub_service.save_epc_property_data" + ) as mock_save, + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + service.run(make_request(uprn="12345")) + + # Assert + coord_client.get_rdsap_summary_by_job_id.assert_called_once() + saved: EpcPropertyData = mock_save.call_args[0][1] + assert saved.energy_rating_current == 78 From e31d2ad2d13f6fb4c32fe26d11d5a5e569e4dc6b Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:49:38 +0000 Subject: [PATCH 10/12] =?UTF-8?q?Preserve=20SharePoint=20distribution=20wh?= =?UTF-8?q?en=20the=20site-note=20save=20fails=20loudly=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_pashub_service.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index aa8b6e2ce..1acb583d3 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -896,3 +896,34 @@ def test_run_fetches_summary_from_coordination_client_on_primary_auth_failure() coord_client.get_rdsap_summary_by_job_id.assert_called_once() saved: EpcPropertyData = mock_save.call_args[0][1] assert saved.energy_rating_current == 78 + + +def test_run_uploads_to_sharepoint_even_when_summary_fetch_fails() -> None: + # Arrange — file distribution must survive a loud site-note save failure + mock_client = _site_note_client() + mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down") + + mock_sharepoint = MagicMock(spec=DomnaSharepointClient) + mock_sharepoint.get_folders_in_path.return_value = { + "value": [{"name": "123 Main St"}] + } + service = make_service(pashub_client=mock_client, sharepoint_client=mock_sharepoint) + + # Act / Assert + with ( + patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("backend.pashub_fetcher.pashub_service.db_session"), + patch("backend.pashub_fetcher.pashub_service.os.remove"), + ): + with pytest.raises(RuntimeError): + service.run( + make_request( + uprn="12345", + sharepoint_link="Retrofit/Properties", + address="123 Main St | deal", + ) + ) + + mock_sharepoint.upload_file.assert_any_call( + RD_SAP_SITE_NOTE_PATH, "Retrofit/Properties/123 Main St", "RdSAP_SiteNote_001.pdf" + ) From 38cae15f4d84312078ef39c0869e45175cb0547c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 13:50:31 +0000 Subject: [PATCH 11/12] =?UTF-8?q?Preserve=20SharePoint=20distribution=20wh?= =?UTF-8?q?en=20the=20site-note=20save=20fails=20loudly=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/pashub_service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 02ca24003..18caab333 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -119,6 +119,7 @@ class PashubService: job_id, include_other=request.get_other_files ) + upload_records: List[_FileUploadRecord] = [] if uprn or hubspot_deal_id: logger.info("Uploading files to s3") file_source = ( @@ -129,7 +130,6 @@ class PashubService: upload_records = self._upload_to_s3_and_update_db( downloaded.core, uprn, hubspot_deal_id, file_source ) - self._save_site_notes(upload_records, active_client, job_id) if downloaded.other: self._upload_to_s3_and_update_db( @@ -153,6 +153,11 @@ class PashubService: property_folder_path = f"{request.sharepoint_link}/{match}" self._upload_to_sharepoint(property_folder_path, downloaded.core + downloaded.other) + # Ordered after S3 + SharePoint so a loud RdSAP Summary failure cannot + # cost us file distribution: the Property refuses to persist with an + # incomplete Lodged Performance, but the evidence is already delivered. + self._save_site_notes(upload_records, active_client, job_id) + for df in downloaded.core + downloaded.other: try: os.remove(df.file_path) From 442352c37851892f3a34e44afe055bf9293d92e0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 14 Jul 2026 14:07:23 +0000 Subject: [PATCH 12/12] =?UTF-8?q?Drop=20the=20unused=20DownloadedFile=20im?= =?UTF-8?q?port=20from=20the=20client=20tests=20=F0=9F=9F=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/pashub_fetcher/tests/test_pashub_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/backend/pashub_fetcher/tests/test_pashub_client.py index 48731eea6..67b2ebac5 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/backend/pashub_fetcher/tests/test_pashub_client.py @@ -9,7 +9,6 @@ from backend.pashub_fetcher.core_files import CoreFiles from backend.pashub_fetcher.evidence_file_data import EvidenceFileData from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata from backend.pashub_fetcher.pashub_client import ( - DownloadedFile, DownloadedFiles, PashubClient, RdSapSummary,