junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-27 23:35:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feature/pashub-normalize-main-fuel-1552

Browse source
This commit is contained in:
Daniel Roth 2026-07-14 14:36:11 +00:00
commit 3c49f4250e
4 changed files with 424 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
backend/pashub_fetcher/pashub_client.py
View file

@ -1,6 +1,6 @@
from collections import defaultdict
import os
from typing import Dict, List, NamedTuple, Optional
from typing import Any, Dict, List, NamedTuple, Optional
from datetime import datetime
import requests
@ -29,6 +29,12 @@ class DownloadedFiles(NamedTuple):
other: List[DownloadedFile]
class RdSapSummary(NamedTuple):
pre_sap_rating: Optional[float]
pre_current_co2_emissions: Optional[float]
pre_current_energy_consumption: Optional[float]
class UnauthorizedError(Exception):
pass
@ -126,6 +132,23 @@ class PashubClient:
)
return None
def get_rdsap_summary_by_job_id(self, job_id: str) -> RdSapSummary:
logger.info(f"Getting RdSAP Summary for job ID {job_id}")
url = f"{self.base}/jobs/{job_id}/rdSapSummary"
r = self.session.get(url)
if r.status_code == 401:
raise UnauthorizedError("Token expired or invalid")
r.raise_for_status()
data: Dict[str, Any] = r.json()
return RdSapSummary(
pre_sap_rating=data.get("preSapRating"),
pre_current_co2_emissions=data.get("preCurrentCo2Emissions"),
pre_current_energy_consumption=data.get("preCurrentEnergyConsumption"),
)
def _group_into_core_and_other_files(
self,
files: List[EvidenceFileData],

66
backend/pashub_fetcher/pashub_service.py
View file

@ -1,6 +1,7 @@
import os
from dataclasses import replace
from datetime import datetime, timezone
from typing import Callable, List, NamedTuple, Optional, cast
from typing import Any, Callable, Dict, List, NamedTuple, Optional, cast
from backend.app.db.connection import db_session
from infrastructure.postgres.uploaded_file_table import (
@ -15,11 +16,13 @@ from backend.pashub_fetcher.pashub_client import (
DownloadedFile,
DownloadedFiles,
PashubClient,
RdSapSummary,
UnauthorizedError,
)
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)
from datatypes.epc.domain.epc import Epc
from datatypes.epc.domain.epc_property_data import EpcPropertyData
from utils.logger import setup_logger
from utils.s3 import upload_file_to_s3
@ -28,6 +31,31 @@ from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
logger = setup_logger()
def _overlay_lodged_performance(
epc_data: EpcPropertyData, summary: RdSapSummary
) -> EpcPropertyData:
"""Fill the as-surveyed current-performance block PasHub carries but the
Site Notes PDF does not. Best-effort per field: values PasHub returns as
null are left empty. The EPC Band is derived from the SAP score via
`Epc.from_sap_score` rather than PasHub's opaque band code."""
updates: Dict[str, Any] = {}
if summary.pre_sap_rating is not None:
score = int(summary.pre_sap_rating)
updates["energy_rating_current"] = score
updates["current_energy_efficiency_band"] = Epc.from_sap_score(score)
if summary.pre_current_co2_emissions is not None:
updates["co2_emissions_current"] = summary.pre_current_co2_emissions
if summary.pre_current_energy_consumption is not None:
updates["energy_consumption_current"] = int(
summary.pre_current_energy_consumption
)
return replace(epc_data, **updates)
class _FileUploadRecord(NamedTuple):
file_path: str
file_type: Optional[str]
@ -91,6 +119,7 @@ class PashubService:
job_id, include_other=request.get_other_files
)
upload_records: List[_FileUploadRecord] = []
if uprn or hubspot_deal_id:
logger.info("Uploading files to s3")
file_source = (
@ -101,7 +130,6 @@ class PashubService:
upload_records = self._upload_to_s3_and_update_db(
downloaded.core, uprn, hubspot_deal_id, file_source
)
self._save_site_notes(upload_records)
if downloaded.other:
self._upload_to_s3_and_update_db(
@ -125,6 +153,11 @@ class PashubService:
property_folder_path = f"{request.sharepoint_link}/{match}"
self._upload_to_sharepoint(property_folder_path, downloaded.core + downloaded.other)
# Ordered after S3 + SharePoint so a loud RdSAP Summary failure cannot
# cost us file distribution: the Property refuses to persist with an
# incomplete Lodged Performance, but the evidence is already delivered.
self._save_site_notes(upload_records, active_client, job_id)
for df in downloaded.core + downloaded.other:
try:
os.remove(df.file_path)
@ -190,17 +223,44 @@ class PashubService:
return upload_records
def _save_site_notes(self, upload_records: List[_FileUploadRecord]) -> None:
def _fetch_rdsap_summary(
self, active_client: PashubClient, job_id: str
) -> RdSapSummary:
try:
return active_client.get_rdsap_summary_by_job_id(job_id)
except UnauthorizedError:
if active_client is not self._pashub_client:
raise
logger.info(
f"PasHub credentials unauthorized for RdSAP Summary on job {job_id}; "
"retrying with CoordinationHub credentials"
)
return self._get_coordination_client().get_rdsap_summary_by_job_id(job_id)
def _save_site_notes(
self,
upload_records: List[_FileUploadRecord],
active_client: PashubClient,
job_id: str,
) -> None:
for record in upload_records:
if (
record.file_type is None
or FileTypeEnum(record.file_type) != FileTypeEnum.RD_SAP_SITE_NOTE
):
continue
# The RdSAP Summary carries the as-surveyed Lodged Performance the
# Site Notes PDF lacks. Fetch it outside the parse try/except so a
# missing summary fails loudly rather than persisting a Property with
# an incomplete Lodged Performance.
summary: RdSapSummary = self._fetch_rdsap_summary(active_client, job_id)
try:
epc_data: EpcPropertyData = parse_site_notes_pdf(
record.file_path, uprn=record.uprn
)
epc_data = _overlay_lodged_performance(epc_data, summary)
with db_session() as session:
save_epc_property_data(
session, epc_data, uploaded_file_id=record.uploaded_file_id

74
backend/pashub_fetcher/tests/test_pashub_client.py
View file

@ -1,14 +1,18 @@
# pyright: reportPrivateUsage=false
from typing import Optional
from unittest.mock import patch
from typing import Any, Optional
from unittest.mock import MagicMock, patch
import pytest
import requests
from backend.pashub_fetcher.core_files import CoreFiles
from backend.pashub_fetcher.evidence_file_data import EvidenceFileData
from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata
from backend.pashub_fetcher.pashub_client import (
DownloadedFile,
DownloadedFiles,
PashubClient,
RdSapSummary,
UnauthorizedError,
)
@ -278,3 +282,67 @@ def test_get_evidence_files_by_job_id_downloads_other_files_when_include_other_t
# Assert
assert [df.file_path for df in result.core] == ["/tmp/SiteNote_001.pdf"]
assert [df.file_path for df in result.other] == ["/tmp/unknown_doc.pdf"]
# ---------------------------------------------------------------------------
# get_rdsap_summary_by_job_id
# ---------------------------------------------------------------------------
def make_response(
status_code: int = 200,
payload: Optional[dict[str, Any]] = None,
raise_on_status: Optional[Exception] = None,
) -> MagicMock:
response = MagicMock()
response.status_code = status_code
response.json.return_value = payload if payload is not None else {}
if raise_on_status is not None:
response.raise_for_status.side_effect = raise_on_status
return response
def test_get_rdsap_summary_maps_pre_performance_fields() -> None:
# Arrange
client = make_client()
payload = {
"preSapRating": 78.00,
"preSapBand": 50,
"preCurrentEnergyConsumption": 117.00,
"preCurrentCo2Emissions": 1.70,
}
# Act
client.session.get = MagicMock(return_value=make_response(payload=payload))
result = client.get_rdsap_summary_by_job_id("job-1")
# Assert
assert result == RdSapSummary(
pre_sap_rating=78.00,
pre_current_co2_emissions=1.70,
pre_current_energy_consumption=117.00,
)
def test_get_rdsap_summary_raises_unauthorized_on_401() -> None:
# Arrange
client = make_client()
# Act / Assert
client.session.get = MagicMock(return_value=make_response(status_code=401))
with pytest.raises(UnauthorizedError):
client.get_rdsap_summary_by_job_id("job-1")
def test_get_rdsap_summary_raises_on_http_error() -> None:
# Arrange
client = make_client()
# Act / Assert — a 404/error must raise loudly, not resolve to empty values
client.session.get = MagicMock(
return_value=make_response(
status_code=404, raise_on_status=requests.HTTPError("not found")
)
)
with pytest.raises(requests.HTTPError):
client.get_rdsap_summary_by_job_id("job-1")

275
backend/pashub_fetcher/tests/test_pashub_service.py
View file

@ -1,5 +1,5 @@
import pytest
from datetime import datetime
from datetime import date, datetime
from typing import Any, Callable, Optional
from unittest.mock import MagicMock, call, patch
@ -9,16 +9,74 @@ from backend.pashub_fetcher.pashub_client import (
DownloadedFile,
DownloadedFiles,
PashubClient,
RdSapSummary,
UnauthorizedError,
)
from backend.pashub_fetcher.pashub_service import PashubService
from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
PashubToAraTriggerRequest,
)
from datatypes.epc.domain.epc import Epc
from datatypes.epc.domain.epc_property_data import (
EpcPropertyData,
SapEnergySource,
SapHeating,
)
from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
FAKE_JOB_LINK = "https://pashub.net/jobs/job-id-123/details"
RD_SAP_SITE_NOTE_PATH = "/tmp/RdSAP_SiteNote_001.pdf"
def make_epc_property_data() -> EpcPropertyData:
"""A minimal, real EpcPropertyData whose as-surveyed performance block is
empty — mirroring a freshly parsed Site Notes PDF the overlay fills in."""
return EpcPropertyData(
dwelling_type="House",
inspection_date=date(2024, 1, 1),
tenure="1",
transaction_type="1",
address_line_1="1 Test Street",
postcode="AB1 2CD",
post_town="Testville",
roofs=[],
walls=[],
floors=[],
main_heating=[],
door_count=0,
sap_heating=SapHeating(
instantaneous_wwhrs=None,
main_heating_details=[],
has_fixed_air_conditioning=False,
),
sap_windows=[],
sap_energy_source=SapEnergySource(
gas_connection_available=False,
meter_type="Single",
pv_battery_count=0,
wind_turbines_count=0,
gas_smart_meter_present=False,
is_dwelling_export_capable=False,
wind_turbines_terrain_type="Suburban",
electricity_smart_meter_present=False,
),
sap_building_parts=[],
solar_water_heating=False,
has_hot_water_cylinder=False,
has_fixed_air_conditioning=False,
wet_rooms_count=0,
extensions_count=0,
heated_rooms_count=0,
open_chimneys_count=0,
habitable_rooms_count=0,
insulated_door_count=0,
cfl_fixed_lighting_bulbs_count=0,
led_fixed_lighting_bulbs_count=0,
incandescent_fixed_lighting_bulbs_count=0,
total_floor_area_m2=80.0,
)
def make_request(
pashub_link: str = FAKE_JOB_LINK,
@ -232,13 +290,15 @@ def test_run_uses_hubspot_deal_id_path_when_no_uprn() -> None:
def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None:
mock_client = MagicMock(spec=PashubClient)
mock_client.get_uprn_by_job_id.return_value = None
mock_client.get_evidence_files_by_job_id.return_value = make_downloaded(
core=["/tmp/RdSAP_SiteNote_001.pdf"]
mock_client = _site_note_client(
summary=RdSapSummary(
pre_sap_rating=None,
pre_current_co2_emissions=None,
pre_current_energy_consumption=None,
)
)
fake_epc_data = MagicMock()
parsed = make_epc_property_data()
fake_session = MagicMock()
fake_uploaded_file_id = 99
@ -248,7 +308,7 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None:
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch(
"backend.pashub_fetcher.pashub_service.parse_site_notes_pdf",
return_value=fake_epc_data,
return_value=parsed,
) as mock_parse,
patch(
"backend.pashub_fetcher.pashub_service.save_epc_property_data"
@ -262,9 +322,9 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None:
mock_db.return_value.__enter__.return_value = fake_session
service.run(make_request(uprn="12345"))
mock_parse.assert_called_once_with("/tmp/RdSAP_SiteNote_001.pdf")
mock_parse.assert_called_once_with(RD_SAP_SITE_NOTE_PATH, uprn=12345)
mock_save.assert_called_once_with(
fake_session, fake_epc_data, uploaded_file_id=fake_uploaded_file_id
fake_session, parsed, uploaded_file_id=fake_uploaded_file_id
)
@ -670,3 +730,200 @@ def test_run_warns_and_continues_when_site_notes_parsing_fails() -> None:
assert result == ["/tmp/RdSAP_SiteNote_001.pdf"]
mock_logger.warning.assert_called()
mock_save.assert_not_called()
# ---------------------------------------------------------------------------
# run(): RdSAP Summary overlaid onto Site-Notes Lodged Performance
# ---------------------------------------------------------------------------
def _site_note_client(
summary: Any = None,
core: Optional[list[str]] = None,
) -> MagicMock:
mock_client = MagicMock(spec=PashubClient)
mock_client.get_uprn_by_job_id.return_value = None
mock_client.get_evidence_files_by_job_id.return_value = make_downloaded(
core=core if core is not None else [RD_SAP_SITE_NOTE_PATH]
)
mock_client.get_rdsap_summary_by_job_id.return_value = summary
return mock_client
def test_run_overlays_full_lodged_performance_onto_saved_site_notes() -> None:
# Arrange
mock_client = _site_note_client(
summary=RdSapSummary(
pre_sap_rating=78.0,
pre_current_co2_emissions=1.70,
pre_current_energy_consumption=117.0,
)
)
parsed = make_epc_property_data()
service = make_service(pashub_client=mock_client)
# Act
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch(
"backend.pashub_fetcher.pashub_service.parse_site_notes_pdf",
return_value=parsed,
),
patch(
"backend.pashub_fetcher.pashub_service.save_epc_property_data"
) as mock_save,
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
service.run(make_request(uprn="12345"))
# Assert
saved: EpcPropertyData = mock_save.call_args[0][1]
assert saved.energy_rating_current == 78
assert saved.current_energy_efficiency_band == Epc.C
assert saved.co2_emissions_current == 1.70
assert saved.energy_consumption_current == 117
def test_run_overlays_only_present_fields_when_summary_partial() -> None:
# Arrange — only the SAP rating comes back; CO2 and PEUI are null
mock_client = _site_note_client(
summary=RdSapSummary(
pre_sap_rating=78.0,
pre_current_co2_emissions=None,
pre_current_energy_consumption=None,
)
)
parsed = make_epc_property_data()
service = make_service(pashub_client=mock_client)
# Act
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch(
"backend.pashub_fetcher.pashub_service.parse_site_notes_pdf",
return_value=parsed,
),
patch(
"backend.pashub_fetcher.pashub_service.save_epc_property_data"
) as mock_save,
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
service.run(make_request(uprn="12345"))
# Assert
saved: EpcPropertyData = mock_save.call_args[0][1]
assert saved.energy_rating_current == 78
assert saved.current_energy_efficiency_band == Epc.C
assert saved.co2_emissions_current is None
assert saved.energy_consumption_current is None
def test_run_raises_and_skips_persistence_when_summary_fetch_fails() -> None:
# Arrange — the RdSAP Site Note is present but its summary cannot be fetched
mock_client = _site_note_client()
mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down")
service = make_service(pashub_client=mock_client)
# Act / Assert
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3,
patch(
"backend.pashub_fetcher.pashub_service.save_epc_property_data"
) as mock_save,
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
with pytest.raises(RuntimeError):
service.run(make_request(uprn="12345"))
mock_save.assert_not_called()
mock_s3.assert_called() # S3 upload happened before the loud site-note save
def test_run_does_not_fetch_summary_for_evidence_only_job() -> None:
# Arrange — a plain Site Note (not an RdSAP Site Note) carries no summary
mock_client = _site_note_client(core=["/tmp/SiteNote_001.pdf"])
service = make_service(pashub_client=mock_client)
# Act
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
service.run(make_request(uprn="12345"))
# Assert
mock_client.get_rdsap_summary_by_job_id.assert_not_called()
def test_run_fetches_summary_from_coordination_client_on_primary_auth_failure() -> None:
# Arrange — primary credentials 401 on the summary; coordination succeeds
pas_client = _site_note_client()
pas_client.get_rdsap_summary_by_job_id.side_effect = UnauthorizedError()
coord_client = MagicMock(spec=PashubClient)
coord_client.get_rdsap_summary_by_job_id.return_value = RdSapSummary(
pre_sap_rating=78.0,
pre_current_co2_emissions=1.70,
pre_current_energy_consumption=117.0,
)
factory = MagicMock(return_value=coord_client)
parsed = make_epc_property_data()
service = make_service(
pashub_client=pas_client, coordination_client_factory=factory
)
# Act
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch(
"backend.pashub_fetcher.pashub_service.parse_site_notes_pdf",
return_value=parsed,
),
patch(
"backend.pashub_fetcher.pashub_service.save_epc_property_data"
) as mock_save,
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
service.run(make_request(uprn="12345"))
# Assert
coord_client.get_rdsap_summary_by_job_id.assert_called_once()
saved: EpcPropertyData = mock_save.call_args[0][1]
assert saved.energy_rating_current == 78
def test_run_uploads_to_sharepoint_even_when_summary_fetch_fails() -> None:
# Arrange — file distribution must survive a loud site-note save failure
mock_client = _site_note_client()
mock_client.get_rdsap_summary_by_job_id.side_effect = RuntimeError("summary down")
mock_sharepoint = MagicMock(spec=DomnaSharepointClient)
mock_sharepoint.get_folders_in_path.return_value = {
"value": [{"name": "123 Main St"}]
}
service = make_service(pashub_client=mock_client, sharepoint_client=mock_sharepoint)
# Act / Assert
with (
patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"),
patch("backend.pashub_fetcher.pashub_service.db_session"),
patch("backend.pashub_fetcher.pashub_service.os.remove"),
):
with pytest.raises(RuntimeError):
service.run(
make_request(
uprn="12345",
sharepoint_link="Retrofit/Properties",
address="123 Main St | deal",
)
)
mock_sharepoint.upload_file.assert_any_call(
RD_SAP_SITE_NOTE_PATH, "Retrofit/Properties/123 Main St", "RdSAP_SiteNote_001.pdf"
)