Compare commits

...

20 commits

Author SHA1 Message Date
Jun-te Kim
5ec7cd83a9
Merge c9a9620527 into 6706e76a28 2026-06-01 15:00:49 +01:00
Jun-te Kim
c9a9620527 pr review, move domain and orhcestration 2026-06-01 14:00:31 +00:00
Jun-te Kim
9c1b6c76a9 delete playground 2026-06-01 12:08:29 +00:00
Jun-te Kim
7f2f2b95a0 update tests to reflect wall types 2026-06-01 09:34:35 +00:00
KhalimCK
6706e76a28
Merge pull request #1121 from Hestia-Homes/feature/solar-api-client
GoogleSolarApiClient class
2026-05-30 15:21:50 +01:00
Jun-te Kim
233cae5eac
Merge pull request #1126 from Hestia-Homes/feature/additional_hubspot_data
hubspot projects data is scraped
2026-05-28 15:28:57 +01:00
Jun-te Kim
17016b299a booking status 2026-05-28 12:15:37 +00:00
Jun-te Kim
2b29b4b458 hubspot projects data is scraped 2026-05-28 10:54:15 +00:00
Daniel Roth
dc5fcc41bf
Merge pull request #1119 from Hestia-Homes/claude/model-p4
Solar API Client
2026-05-21 17:09:17 +01:00
Daniel Roth
0d4462d131 GoogleSolarApi translates BuildingInsightsNotFoundError to sentinel dict 🟩 2026-05-21 16:06:54 +00:00
Daniel Roth
50e9d88752 GoogleSolarApi delegates get_building_insights to GoogleSolarApiClient 🟩 2026-05-21 16:00:59 +00:00
Daniel Roth
521294ad91 GoogleSolarApi delegates get_building_insights to GoogleSolarApiClient 🟥 2026-05-21 15:59:22 +00:00
Daniel Roth
b1933c07c3 GoogleSolarApiClient propagates exception after retry exhaustion 🟩 2026-05-21 15:53:39 +00:00
Daniel Roth
d1ca9be580 GoogleSolarApiClient raises BuildingInsightsNotFoundError on 404 entity-not-found 🟩 2026-05-21 15:52:52 +00:00
Daniel Roth
573656be64 GoogleSolarApiClient raises BuildingInsightsNotFoundError on 404 entity-not-found 🟥 2026-05-21 15:52:21 +00:00
Daniel Roth
497ef1faed GoogleSolarApiClient retries on transient HTTP errors 🟩 2026-05-21 15:51:01 +00:00
Daniel Roth
44217bf361 GoogleSolarApiClient retries on transient HTTP errors 🟥 2026-05-21 15:49:57 +00:00
Daniel Roth
b0106fa93d GoogleSolarApiClient fetches building insights from the Solar API 🟩 2026-05-21 15:48:45 +00:00
Daniel Roth
629fc34a0f GoogleSolarApiClient fetches building insights from the Solar API 🟥 2026-05-21 15:46:47 +00:00
Daniel Roth
2906a60bbb Merge branch 'main' into feature/solar-api-client 2026-05-21 15:30:57 +00:00
42 changed files with 537 additions and 219 deletions

View file

@ -27,3 +27,4 @@ pytest-postgresql
# Formatting
black==26.1.0
boto3-stubs
openai

View file

@ -7,11 +7,11 @@ import boto3
from applications.landlord_description_overrides.landlord_description_overrides_trigger_body import (
LandlordDescriptionOverridesTriggerBody,
)
from domain.landlord_description_overrides.built_form_type import BuiltFormType
from domain.landlord_description_overrides.property_type import PropertyType
from domain.landlord_description_overrides.roof_type import RoofType
from domain.landlord_description_overrides.wall_type import WallType
from domain.landlord_description_overrides.wall_type_construction_dates import (
from domain.epc.built_form_type import BuiltFormType
from domain.epc.property_type import PropertyType
from domain.epc.roof_type import RoofType
from domain.epc.wall_type import WallType
from domain.epc.wall_type_construction_dates import (
wall_type_construction_date_prompt_hint,
)
from infrastructure.chatgpt.chatgpt import ChatGPT

View file

@ -1,13 +1,15 @@
import time
import requests
import pandas as pd
import numpy as np
from typing import List
from typing import Any, List
from functools import lru_cache
from sklearn.preprocessing import MinMaxScaler
from tqdm import tqdm
from math import sin, cos, sqrt, atan2, radians
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
from utils.logger import setup_logger
from recommendations.Costs import Costs
from backend.ml_models.AnnualBillSavings import AnnualBillSavings
@ -57,19 +59,9 @@ class GoogleSolarApi:
# that we calcualte based on the property dimensions, we will correct the roof area
ROOF_AREA_TOLERANCE = 1.25
# Error Messages
ENTITY_NOT_FOUND_ERROR = 'Requested entity was not found.'
def __init__(self, api_key, solar_materials: list, max_retries=5):
"""
Initialize the GoogleSolarApi class with the provided API key and maximum retries.
:param api_key: The API key to authenticate requests to the Google Solar API.
:param max_retries: The maximum number of retries for the API request (default is 5).
"""
def __init__(self, api_key: str, solar_materials: list) -> None:
self.api_key = api_key
self.max_retries = max_retries
self.base_url = "https://solar.googleapis.com/v1"
self._solar_client = GoogleSolarApiClient(api_key)
self.insights_data = None
self.roof_segments = []
@ -90,48 +82,11 @@ class GoogleSolarApi:
self.allowed_segment_indices = None
def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None):
"""
Make an API request to retrieve building insights based on the given longitude and latitude, with retry
mechanism.
:param longitude: The longitude of the location.
:param latitude: The latitude of the location.
:param required_quality: The required quality of the data (default is "MEDIUM").
:param max_retries: The maximum number of retries for the API request (default is None, which uses the
instance's max_retries).
:return: The JSON response containing the building insights data.
"""
if max_retries is None:
max_retries = self.max_retries
insights_url = f"{self.base_url}/buildingInsights:findClosest"
params = {
'location.latitude': f'{latitude:.5f}',
'location.longitude': f'{longitude:.5f}',
'requiredQuality': required_quality,
'key': self.api_key
}
attempt = 0
while attempt < max_retries:
try:
response = requests.get(insights_url, params=params)
response.raise_for_status() # Raise an error for bad status codes
return response.json()
except requests.exceptions.RequestException as e:
if (
(e.response.status_code == 404) &
(e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR)
):
logger.warning("No building insights found for the given location.")
return {"error": self.ENTITY_NOT_FOUND_ERROR}
attempt += 1
print(f"Attempt {attempt} failed: {e}")
time.sleep(2 ** attempt) # Exponential backoff
if attempt >= max_retries:
raise
def get_building_insights(self, longitude: float, latitude: float, required_quality: str = "MEDIUM") -> dict[str, Any]:
try:
return self._solar_client.get_building_insights(longitude, latitude, required_quality) # type: ignore[arg-type]
except BuildingInsightsNotFoundError:
return {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}
@lru_cache(maxsize=128)
def get(

View file

@ -17,12 +17,14 @@ class HubspotDealData(SQLModel, table=True):
dealstage: Optional[str] = Field(default=None)
company_id: Optional[str] = Field(default=None)
project_code: Optional[str] = Field(default=None)
project_id: Optional[str] = Field(default=None)
# HubSpot custom properties
landlord_property_id: Optional[str] = Field(default=None)
uprn: Optional[str] = Field(default=None)
outcome: Optional[str] = Field(default=None)
outcome_notes: Optional[str] = Field(default=None)
booking_status: Optional[str] = Field(default=None)
major_condition_issue_description: Optional[str] = Field(default=None)
major_condition_issue_photos: Optional[str] = Field(default=None)

View file

@ -0,0 +1,34 @@
import uuid
from sqlmodel import SQLModel, Field, Column, text
from datetime import datetime
from typing import Optional
from sqlalchemy import DateTime
from sqlalchemy.sql import func
class HubspotProjectData(SQLModel, table=True):
__tablename__ = "hubspot_projects_data"
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
project_id: str = Field(index=True, nullable=False, unique=True)
name: Optional[str] = Field(default=None)
created_at: Optional[datetime] = Field(
sa_column=Column(
DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"),
nullable=False,
),
default=func.now(),
)
updated_at: Optional[datetime] = Field(
sa_column=Column(
DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"),
onupdate=func.now(),
nullable=False,
),
default=func.now(),
)

View file

@ -14,55 +14,11 @@ class TestSearchEpcIntegration:
def epc_auth_token(self):
return os.getenv("EPC_AUTH_TOKEN")
@pytest.mark.parametrize(
"address, postcode, uprn, skip_os, lmk_key, n_old_epcs",
[
# Test case 1: Valid address and postcode, skipping OS
# In this case, the property is an individual flat but the uprn associated to the
# EPC is for the building as a whole, possibly because there was a conversion of sorts
("Garden Flat, 48 Bedminster Parade", "BS3 4HS", 308249, True,
"260907a5431fa073d193cc6bbec51fbf1ba9a61845ab2503f85aa19ce3ed6afd", 1),
# Test case 2: Another valid address and postcode
# In this case, the newest EPC, does not have a uprn associated to it. If we did a search by
# uprn, we would get an old EPC
("Flat 8, Hainton House", "DN32 9AQ", "", True,
"bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 2),
# Test case 3: When we make a request to the API for this property, we get back results for
# flats 1, 2 and 3. We have some logic to handle the response so that we get back flat 1
("Flat 1, 1 Tottenham Street, London", "W1T 2AE", 5167411, True,
"3e6414d7f15f4cf7a69dc20c469bcf043d31a49239b183f1bd0c0e1aafa23c93", 0),
],
)
def test_find_property(self, epc_auth_token, address, postcode, uprn, skip_os, lmk_key, n_old_epcs):
"""
Integration test for `find_property`, making actual API calls.
"""
# Provide your actual API keys or tokens here
os_api_key = ""
# Initialize the SearchEpc instance
epc_searcher = SearchEpc(
address1=address,
postcode=postcode,
uprn=uprn,
auth_token=epc_auth_token,
os_api_key=os_api_key,
)
# Execute the method
epc_searcher.find_property(skip_os=skip_os)
# We check that we have the correct epc
assert epc_searcher.newest_epc["lmk-key"] == lmk_key
assert len(epc_searcher.older_epcs) == n_old_epcs
def test_search_housenumber(self):
eg1 = 'Flat A11, Mortimer House, Grendon Road, Exeter'
eg1 = "Flat A11, Mortimer House, Grendon Road, Exeter"
res1 = SearchEpc.get_house_number(eg1, None)
assert res1 == "A11"
eg2 = 'Flat A9, Mortimer House, Grendon Road, Exeter, EX1 2NL'
eg2 = "Flat A9, Mortimer House, Grendon Road, Exeter, EX1 2NL"
res2 = SearchEpc.get_house_number(eg2, None)
assert res2 == "A9"

View file

@ -27,7 +27,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Mapping, Optional
from domain.landlord_description_overrides.wall_type import WallType
from domain.epc.wall_type import WallType
@dataclass(frozen=True)

View file

@ -31,6 +31,7 @@ from hubspot.crm.associations.v4.models import ( # type: ignore[reportMissingTy
from backend.app.config import get_settings
from etl.hubspot.company_data import CompanyData
from etl.hubspot.project_data import ProjectData
from utils.logger import setup_logger
import mimetypes
@ -239,6 +240,47 @@ class HubspotClient:
self.logger.info(f"Listing info for deal {deal_id}: {listing_info}")
return listing_info
def from_deal_id_get_associated_project(
self, deal_id: str
) -> Optional[ProjectData]:
"""
Get the associated project (custom object "0-970") for a given deal.
Returns a ProjectData with "project_id" and "name", or None if not found.
"""
associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api # type: ignore[reportUnknownMemberType]
projects_api: ObjectsBasicApi = self.client.crm.objects.basic_api # type: ignore[reportUnknownMemberType] # works for custom objects like "project"
response: AssociationsPageResponse = self._call_with_retry(
lambda: associations_api.get_page( # type: ignore[reportUnknownMemberType]
object_type="deals",
object_id=deal_id,
to_object_type="0-970",
limit=1,
)
)
results: list[AssociationsResult] = cast(list[AssociationsResult], response.results) # type: ignore[reportUnknownMemberType]
if not results:
self.logger.info(f"No project association found for deal {deal_id}")
return None
first: AssociationsResult = results[0]
project_id: str = cast(str, first.to_object_id) # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
self.logger.info(f"Associated project ID for deal {deal_id}: {project_id}")
project: HubspotObject = self._call_with_retry(
lambda: projects_api.get_by_id( # type: ignore[reportUnknownMemberType]
object_type="0-970",
object_id=project_id,
properties=["hs_name"],
)
)
project_properties: dict[str, str] = cast(dict[str, str], project.properties) # type: ignore[reportUnknownMemberType]
return ProjectData(
project_id=project_id, name=project_properties.get("hs_name")
)
def from_deal_id_get_info(
self, deal_id: str
) -> dict[str, str]: # TODO: add dataclass for this
@ -253,6 +295,7 @@ class HubspotClient:
"pipeline",
"outcome",
"outcome_notes",
"booking_status",
"project_code",
"major_condition_issue_description",
"major_condition_issue_photos",
@ -325,17 +368,22 @@ class HubspotClient:
self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}")
return None
def get_deal_and_company_and_listing(
def get_deal_and_company_and_listing_and_project(
self, deal_id: str
) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]:
) -> tuple[
dict[str, str], Optional[str], Optional[dict[str, str]], Optional[ProjectData]
]:
deal: dict[str, str] = self.from_deal_id_get_info(deal_id)
company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id)
listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing(
deal_id
)
project: Optional[ProjectData] = self.from_deal_id_get_associated_project(
deal_id
)
return deal, company, listing
return deal, company, listing, project
def get_company_information(self, company_id: str) -> CompanyData:
companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api # type: ignore[reportUnknownMemberType]

View file

@ -4,9 +4,11 @@ from datetime import datetime, timezone
from typing import Dict, Optional
from backend.app.db.models.hubspot_deal_data import HubspotDealData
from backend.app.db.models.hubspot_project_data import HubspotProjectData
from backend.app.db.models.hubspot_user import HubspotUser
from etl.hubspot.company_data import CompanyData
from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.project_data import ProjectData
from etl.hubspot.s3_uploader import S3Uploader
from backend.app.db.connection import db_read_session
from backend.app.db.models.organisation import Organisation
@ -64,6 +66,30 @@ class HubspotDataToDb:
session.commit()
return record
def upsert_project(self, project: ProjectData) -> HubspotProjectData:
"""Upserts a project record. Updates if project_id exists, otherwise creates new."""
with db_read_session() as session:
project_id = project["project_id"]
name = project.get("name")
existing = session.exec(
select(HubspotProjectData).where(
HubspotProjectData.project_id == project_id
)
).first()
if existing:
existing.name = name
existing.updated_at = datetime.now(timezone.utc)
session.add(existing)
record = existing
else:
record = HubspotProjectData(project_id=project_id, name=name)
session.add(record)
session.commit()
return record
def find_all_deals_with_company_id(self, company_id: str):
"""Returns a list of deals for a given company_id."""
with db_read_session() as session:
@ -87,6 +113,7 @@ class HubspotDataToDb:
company: Optional[str],
listing: Optional[dict[str, str]],
hubspot_client: HubspotClient,
project: Optional[ProjectData] = None,
):
"""
Inserts or updates a deal record.
@ -111,7 +138,9 @@ class HubspotDataToDb:
self._handle_existing_photo_upload(existing, hubspot_client)
print(f"🔄 Updating existing deal (deal_id={deal_id})")
self._update_existing_deal(existing, deal_data, listing, company)
self._update_existing_deal(
existing, deal_data, listing, company, project
)
session.add(existing)
session.commit()
@ -121,7 +150,7 @@ class HubspotDataToDb:
else:
print(f"🆕 Inserting new deal (deal_id={deal_id})")
new_record: HubspotDealData = self._build_new_deal(
deal_id, deal_data, listing, company
deal_id, deal_data, listing, company, project
)
# Handle upload at insert time
@ -170,10 +199,12 @@ class HubspotDataToDb:
deal_data: Dict[str, str],
listing: Optional[dict[str, str]],
company: Optional[str],
project: Optional[ProjectData] = None,
):
for attr, value in {
"dealname": deal_data.get("dealname"),
"dealstage": deal_data.get("dealstage"),
"project_id": project["project_id"] if project else None,
"listing_id": listing.get("listing_id", None) if listing else None,
"landlord_property_id": (
listing.get("owner_property_id", None) if listing else None
@ -181,6 +212,7 @@ class HubspotDataToDb:
"uprn": listing.get("national_uprn", None) if listing else None,
"outcome": deal_data.get("outcome"),
"outcome_notes": deal_data.get("outcome_notes"),
"booking_status": deal_data.get("booking_status"),
"project_code": deal_data.get("project_code"),
"company_id": company,
"major_condition_issue_description": deal_data.get(
@ -270,11 +302,13 @@ class HubspotDataToDb:
deal_data: Dict[str, str],
listing: Optional[dict[str, str]],
company: Optional[str],
project: Optional[ProjectData] = None,
) -> HubspotDealData:
return HubspotDealData(
deal_id=deal_id,
dealname=deal_data.get("dealname"),
dealstage=deal_data.get("dealstage"),
project_id=project["project_id"] if project else None,
listing_id=listing.get("listing_id") if listing else None,
landlord_property_id=(
listing.get("owner_property_id") if listing else None
@ -282,6 +316,7 @@ class HubspotDataToDb:
uprn=listing.get("national_uprn") if listing else None,
outcome=deal_data.get("outcome"),
outcome_notes=deal_data.get("outcome_notes"),
booking_status=deal_data.get("booking_status"),
project_code=deal_data.get("project_code"),
company_id=company,
major_condition_issue_description=deal_data.get(

View file

@ -53,6 +53,7 @@ class HubspotDealDiffer:
"dealname": "dealname",
"project_code": "project_code",
"outcome_notes": "outcome_notes",
"booking_status": "booking_status",
"major_condition_issue_description": "major_condition_issue_description",
"major_condition_issue_photos": "major_condition_issue_photos",
"coordination_status__stage_1_": "coordination_status",

View file

@ -0,0 +1,6 @@
from typing import Optional, TypedDict
class ProjectData(TypedDict):
project_id: str
name: Optional[str]

View file

@ -14,7 +14,7 @@ payload = {
{
"task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917",
"sub_task_id": "8673913b-1a88-42d7-8578-0449123d94b0",
"hubspot_deal_id": "254427203793",
"hubspot_deal_id": "467396027619",
}
)
}

View file

@ -5,6 +5,7 @@ from typing import Any, Dict, Optional
from backend.app.config import get_settings
from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
from etl.hubspot.project_data import ProjectData
from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import (
HubspotTriggerOrchestratorTriggerRequest,
@ -34,9 +35,10 @@ def handler(body: dict[str, Any], context: Any) -> None:
hubspot_deal: Dict[str, str]
company: Optional[str]
listing: Optional[dict[str, str]]
project: Optional[ProjectData]
hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing(
hubspot_deal_id
hubspot_deal, company, listing, project = (
hubspot_client.get_deal_and_company_and_listing_and_project(hubspot_deal_id)
)
deal_changed = False
@ -47,7 +49,10 @@ def handler(body: dict[str, Any], context: Any) -> None:
db_client: HubspotDataToDb = HubspotDataToDb()
db_client.upsert_organisation(company_data)
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
if project:
db_client.upsert_project(project)
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client, project)
# ==============================
# Orchestration of other lambdas
@ -77,11 +82,15 @@ def handler(body: dict[str, Any], context: Any) -> None:
logger.info(
f"Deal {hubspot_deal_id} has been changed, updating database..."
)
if project:
db_client.upsert_project(project)
db_client.upsert_deal(
deal_data=hubspot_deal,
company=company,
listing=listing,
hubspot_client=hubspot_client,
project=project,
)
deal_changed = True

View file

@ -71,7 +71,9 @@ class TestHubspotClientIntegration:
def test_get_deal_info_for_db(self, client: HubspotClient):
deal_id: str = "263490768079"
deal, company, listing = client.get_deal_and_company_and_listing(deal_id)
deal, company, listing, project = (
client.get_deal_and_company_and_listing_and_project(deal_id)
)
assert "dealname" in deal
assert "dealstage" in deal
@ -81,6 +83,8 @@ class TestHubspotClientIntegration:
assert listing is None or "hs_object_id" in listing
assert project is None or "project_id" in project
def test_get_company_information(self, client: HubspotClient):
company_id: str = Companies.APPLE.value

View file

@ -1,4 +1,5 @@
from etl.hubspot.hubspotDataTodB import HubspotDataToDb
from etl.hubspot.project_data import ProjectData
from backend.app.db.models.hubspot_deal_data import HubspotDealData
@ -32,3 +33,55 @@ def test_update_existing_deal__designer_set__overwrites_existing() -> None:
)
assert existing.designer == "New Designer"
def test_build_new_deal__project_sets_project_id() -> None:
new_deal = _make_instance()._build_new_deal(
deal_id="MOCK_DEAL_ID",
deal_data={},
listing=None,
company=None,
project=ProjectData(project_id="proj-1", name="Project One"),
)
assert new_deal.project_id == "proj-1"
def test_build_new_deal__no_project__project_id_none() -> None:
new_deal = _make_instance()._build_new_deal(
deal_id="MOCK_DEAL_ID",
deal_data={},
listing=None,
company=None,
project=None,
)
assert new_deal.project_id is None
def test_update_existing_deal__project_sets_project_id() -> None:
existing = HubspotDealData(deal_id="MOCK_DEAL_ID")
_make_instance()._update_existing_deal(
existing=existing,
deal_data={},
listing=None,
company=None,
project=ProjectData(project_id="proj-1", name="Project One"),
)
assert existing.project_id == "proj-1"
def test_update_existing_deal__no_project__clears_project_id() -> None:
existing = HubspotDealData(deal_id="MOCK_DEAL_ID", project_id="old-proj")
_make_instance()._update_existing_deal(
existing=existing,
deal_data={},
listing=None,
company=None,
project=None,
)
assert existing.project_id is None

View file

@ -12,6 +12,7 @@ DEAL_ID = "999"
PASHUB_LINK = "https://pashub.example.com/deal/999"
MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev"
PASHUB_QUEUE_URL = "https://sqs.test/pashub"
PROJECT = {"project_id": "proj-1", "name": "Project One"}
def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]:
@ -35,7 +36,8 @@ def run_handler(
hubspot_deal: Dict[str, Any],
db_deal: Optional[HubspotDealData],
listing: Optional[dict],
) -> MagicMock:
project: Optional[dict] = None,
) -> tuple[MagicMock, MagicMock]:
mock_sqs = MagicMock()
mock_sqs.send_message.return_value = {"MessageId": "test-id"}
@ -47,10 +49,12 @@ def run_handler(
):
mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal
mock_db_cls.return_value.upsert_deal.return_value = None
mock_hs_cls.return_value.get_deal_and_company_and_listing.return_value = (
mock_hs = mock_hs_cls.return_value
mock_hs.get_deal_and_company_and_listing_and_project.return_value = (
hubspot_deal,
None,
listing,
project,
)
mock_boto3.client.return_value = mock_sqs
mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL
@ -58,7 +62,7 @@ def run_handler(
handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "")
return mock_sqs
return mock_sqs, mock_db_cls.return_value
# ====================================
@ -72,7 +76,7 @@ def test_new_deal_surveyed__sends_magicplan_sqs() -> None:
listing = {"national_uprn": UPRN}
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing)
# Assert
mock_sqs.send_message.assert_called_once_with(
@ -88,7 +92,7 @@ def test_new_deal_not_surveyed__no_magicplan_sqs() -> None:
hubspot_deal = make_hubspot_deal(outcome="assessed")
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert
mock_sqs.send_message.assert_not_called()
@ -99,7 +103,7 @@ def test_new_deal_surveyed_no_listing__magicplan_sqs_uprn_is_null() -> None:
hubspot_deal = make_hubspot_deal(outcome="surveyed")
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert
mock_sqs.send_message.assert_called_once_with(
@ -122,7 +126,7 @@ def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None:
listing = {"national_uprn": UPRN}
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing)
# Assert
mock_sqs.send_message.assert_called_once_with(
@ -139,7 +143,7 @@ def test_existing_deal_already_surveyed__no_magicplan_sqs() -> None:
hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name")
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert
mock_sqs.send_message.assert_not_called()
@ -155,7 +159,7 @@ def test_new_deal_with_pashub_link__sends_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK)
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert
mock_sqs.send_message.assert_called_once_with(
@ -179,7 +183,7 @@ def test_new_deal_no_pashub_link__no_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal()
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert
mock_sqs.send_message.assert_not_called()
@ -196,7 +200,7 @@ def test_existing_deal_pashub_link_added__sends_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK)
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert
mock_sqs.send_message.assert_called_once_with(
@ -221,7 +225,67 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name")
# Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert
mock_sqs.send_message.assert_not_called()
# ====================================
# PROJECT upsert
# ====================================
def test_new_deal_with_project__upserts_project() -> None:
# Arrange
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_called_once_with(PROJECT)
def test_new_deal_no_project__no_project_upsert() -> None:
# Arrange
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=None
)
# Assert
mock_db.upsert_project.assert_not_called()
def test_existing_deal_changed_with_project__upserts_project() -> None:
# Arrange
db_deal = make_db_deal(outcome="assessed")
hubspot_deal = make_hubspot_deal(outcome="surveyed")
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_called_once_with(PROJECT)
def test_existing_deal_unchanged__no_project_upsert() -> None:
# Arrange: db deal matches hubspot deal, so the differ reports no change
db_deal = make_db_deal(dealname=DEAL_NAME)
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_not_called()
mock_db.upsert_deal.assert_not_called()

View file

@ -4,7 +4,7 @@ import json
from enum import Enum
from typing import Any, Optional, TypeVar
from domain.landlord_description_overrides.column_classifier import (
from infrastructure.column_classifier import (
ClassificationError,
ColumnClassifier,
)

View file

@ -17,7 +17,7 @@ from sqlalchemy import Table
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlmodel import Session
from domain.landlord_description_overrides.built_form_type import BuiltFormType
from domain.epc.built_form_type import BuiltFormType
from infrastructure.postgres.landlord_built_form_type_override_table import (
LandlordBuiltFormTypeOverrideRow,
)

View file

@ -16,7 +16,7 @@ from sqlalchemy import BigInteger, Column, UniqueConstraint
from sqlalchemy import Enum as SAEnum
from sqlmodel import Field, SQLModel
from domain.landlord_description_overrides.built_form_type import BuiltFormType
from domain.epc.built_form_type import BuiltFormType
from infrastructure.postgres.landlord_override_enums import override_source_sa_enum

View file

@ -19,7 +19,7 @@ from sqlalchemy import Table
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlmodel import Session
from domain.landlord_description_overrides.property_type import PropertyType
from domain.epc.property_type import PropertyType
from infrastructure.postgres.landlord_override_enums import OverrideSource
from infrastructure.postgres.landlord_property_type_override_table import (
LandlordPropertyTypeOverrideRow,

View file

@ -14,7 +14,7 @@ from sqlalchemy import BigInteger, Column, UniqueConstraint
from sqlalchemy import Enum as SAEnum
from sqlmodel import Field, SQLModel
from domain.landlord_description_overrides.property_type import PropertyType
from domain.epc.property_type import PropertyType
from infrastructure.postgres.landlord_override_enums import override_source_sa_enum

View file

@ -17,7 +17,7 @@ from sqlalchemy import Table
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlmodel import Session
from domain.landlord_description_overrides.roof_type import RoofType
from domain.epc.roof_type import RoofType
from infrastructure.postgres.landlord_override_enums import OverrideSource
from infrastructure.postgres.landlord_roof_type_override_table import (
LandlordRoofTypeOverrideRow,

View file

@ -16,7 +16,7 @@ from sqlalchemy import BigInteger, Column, UniqueConstraint
from sqlalchemy import Enum as SAEnum
from sqlmodel import Field, SQLModel
from domain.landlord_description_overrides.roof_type import RoofType
from domain.epc.roof_type import RoofType
from infrastructure.postgres.landlord_override_enums import override_source_sa_enum

View file

@ -17,7 +17,7 @@ from sqlalchemy import Table
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlmodel import Session
from domain.landlord_description_overrides.wall_type import WallType
from domain.epc.wall_type import WallType
from infrastructure.postgres.landlord_override_enums import OverrideSource
from infrastructure.postgres.landlord_wall_type_override_table import (
LandlordWallTypeOverrideRow,

View file

@ -16,7 +16,7 @@ from sqlalchemy import BigInteger, Column, UniqueConstraint
from sqlalchemy import Enum as SAEnum
from sqlmodel import Field, SQLModel
from domain.landlord_description_overrides.wall_type import WallType
from domain.epc.wall_type import WallType
from infrastructure.postgres.landlord_override_enums import override_source_sa_enum

View file

View file

@ -0,0 +1,50 @@
import time
from typing import Any, Literal, Optional
import requests
class BuildingInsightsNotFoundError(Exception):
pass
class GoogleSolarApiClient:
base_url: str = "https://solar.googleapis.com/v1"
MAX_RETRIES: int = 5
ENTITY_NOT_FOUND_ERROR: str = "Requested entity was not found."
def __init__(self, api_key: str) -> None:
self._api_key = api_key
def get_building_insights(
self,
longitude: float,
latitude: float,
required_quality: Literal["HIGH", "MEDIUM", "LOW"] = "MEDIUM",
) -> dict[str, Any]:
insights_url = f"{self.base_url}/buildingInsights:findClosest"
params: dict[str, str] = {
"location.latitude": f"{latitude:.5f}",
"location.longitude": f"{longitude:.5f}",
"requiredQuality": required_quality,
"key": self._api_key,
}
last_exc: Optional[Exception] = None
for attempt in range(self.MAX_RETRIES):
try:
response = requests.get(insights_url, params=params)
response.raise_for_status()
result: dict[str, Any] = response.json()
return result
except requests.exceptions.RequestException as e:
if (
e.response is not None
and e.response.status_code == 404
and e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR
):
raise BuildingInsightsNotFoundError() from e
last_exc = e
time.sleep(2 ** attempt)
assert last_exc is not None
raise last_exc

View file

@ -4,7 +4,7 @@ from dataclasses import dataclass
from enum import Enum
from typing import Generic, TypeVar
from domain.landlord_description_overrides.column_classifier import ColumnClassifier
from infrastructure.column_classifier import ColumnClassifier
from repositories.landlord_overrides.landlord_override_repository import (
LandlordOverrideRepository,
)

View file

@ -1,57 +0,0 @@
"""Read a file and return unique values from a chosen column."""
from pathlib import Path
import argparse
import sys
import pandas as pd
def read_file(path: str | Path) -> pd.DataFrame:
path = Path(path)
suffix = path.suffix.lower()
if suffix == ".csv":
return pd.read_csv(path)
if suffix == ".tsv":
return pd.read_csv(path, sep="\t")
if suffix in {".xlsx", ".xls"}:
return pd.read_excel(path)
if suffix == ".parquet":
return pd.read_parquet(path)
if suffix == ".json":
return pd.read_json(path)
raise ValueError(f"Unsupported file type: {suffix}")
def get_unique(path: str | Path, column: str, dropna: bool = True) -> list:
df = read_file(Path(path))
if column not in df.columns:
raise KeyError(f"Column {column!r} not found. Available: {list(df.columns)}")
series = df[column].dropna() if dropna else df[column]
return series.unique().tolist()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--path", default="/workspaces/model/certificates-2026.csv")
parser.add_argument("--column", nargs="walls_description")
parser.add_argument("--keep-na", action="store_true")
args, _ = parser.parse_known_args()
df = read_file(args.path)
if not args.column:
print("Available columns:")
for c in df.columns:
print(f" - {c}")
return 0
column = "wall "
series = df[column] if args.keep_na else df[column].dropna()
for value in series.unique():
print(value)
return 0
if __name__ == "__main__":
sys.exit(main())

View file

@ -4,9 +4,9 @@ from typing import Optional
import pytest
from domain.landlord_description_overrides.column_classifier import ClassificationError
from domain.landlord_description_overrides.property_type import PropertyType
from domain.landlord_description_overrides.wall_type import WallType
from infrastructure.column_classifier import ClassificationError
from domain.epc.property_type import PropertyType
from domain.epc.wall_type import WallType
from infrastructure.chatgpt.chatgpt import ChatGPT
from infrastructure.chatgpt.chatgpt_column_classifier import (
ChatGptColumnClassifier,

View file

View file

@ -0,0 +1,110 @@
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
import requests
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
# ---------------------------------------------------------------------------
# Slice 1: Successful response returns parsed JSON
# ---------------------------------------------------------------------------
def test_get_building_insights_returns_parsed_json() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
mock_response = MagicMock()
mock_response.json.return_value = payload
with patch("requests.get", return_value=mock_response) as mock_get:
mock_response.raise_for_status.return_value = None
# Act
result = client.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
mock_get.assert_called_once()
# ---------------------------------------------------------------------------
# Slice 2: Transient HTTP errors trigger retries
# ---------------------------------------------------------------------------
def test_get_building_insights_retries_on_transient_error() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
transient_error = requests.exceptions.ConnectionError("timeout")
transient_error.response = None # type: ignore[attr-defined]
success_response = MagicMock()
success_response.json.return_value = payload
success_response.raise_for_status.return_value = None
with patch("requests.get", side_effect=[transient_error, success_response]) as mock_get:
with patch("time.sleep"):
# Act
result = client.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
assert mock_get.call_count == 2
# ---------------------------------------------------------------------------
# Slice 3: 404 + entity-not-found raises BuildingInsightsNotFoundError
# ---------------------------------------------------------------------------
def test_get_building_insights_raises_on_entity_not_found() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
not_found_response = MagicMock()
not_found_response.status_code = 404
not_found_response.json.return_value = {
"error": {"message": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}
}
http_error = requests.exceptions.HTTPError("404")
http_error.response = not_found_response
with patch("requests.get") as mock_get:
mock_get.return_value.raise_for_status.side_effect = http_error
mock_get.return_value.response = not_found_response
# Act / Assert
with pytest.raises(BuildingInsightsNotFoundError):
client.get_building_insights(longitude=-0.1278, latitude=51.5074)
assert mock_get.call_count == 1
# ---------------------------------------------------------------------------
# Slice 4: Retry exhaustion propagates the exception to the caller
# ---------------------------------------------------------------------------
def test_get_building_insights_propagates_exception_after_retry_exhaustion() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
error = requests.exceptions.ConnectionError("persistent failure")
error.response = None # type: ignore[attr-defined]
with patch("requests.get", side_effect=error) as mock_get:
with patch("time.sleep"):
# Act / Assert
with pytest.raises(requests.exceptions.ConnectionError):
client.get_building_insights(longitude=-0.1278, latitude=51.5074)
assert mock_get.call_count == GoogleSolarApiClient.MAX_RETRIES

View file

@ -0,0 +1,47 @@
from typing import Any
from unittest.mock import patch
from backend.apis.GoogleSolarApi import GoogleSolarApi
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
# ---------------------------------------------------------------------------
# Slice 1: get_building_insights delegates to _solar_client
# ---------------------------------------------------------------------------
def test_get_building_insights_delegates_to_solar_client() -> None:
# Arrange
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
with patch.object(GoogleSolarApiClient, "get_building_insights", return_value=payload):
api = GoogleSolarApi(api_key="test-key", solar_materials=[])
# Act
result = api.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
# ---------------------------------------------------------------------------
# Slice 2: BuildingInsightsNotFoundError translates to sentinel dict
# ---------------------------------------------------------------------------
def test_get_building_insights_returns_sentinel_on_not_found() -> None:
# Arrange
with patch.object(
GoogleSolarApiClient,
"get_building_insights",
side_effect=BuildingInsightsNotFoundError,
):
api = GoogleSolarApi(api_key="test-key", solar_materials=[])
# Act
result = api.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}

View file

@ -4,10 +4,10 @@ from enum import Enum
from typing import Any, Optional
from domain.addresses.unstandardised_address import AddressList, UnstandardisedAddress
from domain.landlord_description_overrides.built_form_type import BuiltFormType
from domain.landlord_description_overrides.column_classifier import ColumnClassifier
from domain.landlord_description_overrides.property_type import PropertyType
from domain.landlord_description_overrides.wall_type import WallType
from domain.epc.built_form_type import BuiltFormType
from infrastructure.column_classifier import ColumnClassifier
from domain.epc.property_type import PropertyType
from domain.epc.wall_type import WallType
from domain.postcode import Postcode
from orchestration.classifiable_column import ClassifiableColumn
from orchestration.landlord_description_overrides_orchestrator import (

View file

@ -15,7 +15,7 @@ import pytest
from sqlalchemy import Engine
from sqlmodel import Session, select
from domain.landlord_description_overrides.property_type import PropertyType
from domain.epc.property_type import PropertyType
from infrastructure.postgres.landlord_override_enums import OverrideSource
from infrastructure.postgres.landlord_property_type_override_postgres_repository import (
LandlordPropertyTypeOverridePostgresRepository,

View file

@ -14,7 +14,7 @@ import pytest
from sqlalchemy import Engine
from sqlmodel import Session, select
from domain.landlord_description_overrides.wall_type import WallType
from domain.epc.wall_type import WallType
from infrastructure.postgres.landlord_override_enums import OverrideSource
from infrastructure.postgres.landlord_wall_type_override_postgres_repository import (
LandlordWallTypeOverridePostgresRepository,
@ -49,13 +49,13 @@ def test_inserts_a_fresh_row_with_source_classifier(session: Session) -> None:
# act
repo.upsert_all(
portfolio_id=1, descriptions_to_values={"cavity insulated": WallType.CAVITY}
portfolio_id=1, descriptions_to_values={"cavity insulated": WallType.CAVITY_FILLED}
)
session.commit()
# assert
row = _select_row(session, portfolio_id=1, description="cavity insulated")
assert row.value is WallType.CAVITY
assert row.value is WallType.CAVITY_FILLED
assert row.source == OverrideSource.CLASSIFIER
@ -63,19 +63,19 @@ def test_reupsert_overwrites_a_classifier_row(session: Session) -> None:
# arrange: a stale classifier row exists.
repo = LandlordWallTypeOverridePostgresRepository(session)
repo.upsert_all(
portfolio_id=1, descriptions_to_values={"old red brick": WallType.CAVITY}
portfolio_id=1, descriptions_to_values={"old red brick": WallType.CAVITY_FILLED}
)
session.commit()
# act: re-classify with a different category.
repo.upsert_all(
portfolio_id=1, descriptions_to_values={"old red brick": WallType.SOLID_BRICK}
portfolio_id=1, descriptions_to_values={"old red brick": WallType.SOLID_BRICK_AS_BUILT_NO_INSULATION_ASSUMED}
)
session.commit()
# assert: the new classification wins.
row = _select_row(session, portfolio_id=1, description="old red brick")
assert row.value is WallType.SOLID_BRICK
assert row.value is WallType.SOLID_BRICK_AS_BUILT_NO_INSULATION_ASSUMED
assert row.source == OverrideSource.CLASSIFIER
@ -86,7 +86,7 @@ def test_reupsert_does_not_overwrite_a_user_row(session: Session) -> None:
user_row = LandlordWallTypeOverrideRow(
portfolio_id=1,
description="old red brick",
value=WallType.SANDSTONE,
value=WallType.SANDSTONE_AS_BUILT_NO_INSULATION_ASSUMED,
source=OverrideSource.USER,
)
session.add(user_row)
@ -97,13 +97,13 @@ def test_reupsert_does_not_overwrite_a_user_row(session: Session) -> None:
# be silently skipped -- user edits beat classifier reruns.
repo = LandlordWallTypeOverridePostgresRepository(session)
repo.upsert_all(
portfolio_id=1, descriptions_to_values={"old red brick": WallType.SOLID_BRICK}
portfolio_id=1, descriptions_to_values={"old red brick": WallType.SOLID_BRICK_AS_BUILT_NO_INSULATION_ASSUMED}
)
session.commit()
# assert: the user row is unchanged.
row = _select_row(session, portfolio_id=1, description="old red brick")
assert row.value is WallType.SANDSTONE
assert row.value is WallType.SANDSTONE_AS_BUILT_NO_INSULATION_ASSUMED
assert row.source == OverrideSource.USER
@ -114,16 +114,16 @@ def test_upsert_keeps_other_portfolios_descriptions_independent(
# same description for two different portfolios must coexist as two rows.
repo = LandlordWallTypeOverridePostgresRepository(session)
repo.upsert_all(
portfolio_id=1, descriptions_to_values={"old red brick": WallType.CAVITY}
portfolio_id=1, descriptions_to_values={"old red brick": WallType.CAVITY_FILLED}
)
repo.upsert_all(
portfolio_id=2, descriptions_to_values={"old red brick": WallType.SOLID_BRICK}
portfolio_id=2, descriptions_to_values={"old red brick": WallType.SOLID_BRICK_AS_BUILT_NO_INSULATION_ASSUMED}
)
session.commit()
# assert: both rows survive with their own values.
assert _select_row(session, 1, "old red brick").value is WallType.CAVITY
assert _select_row(session, 2, "old red brick").value is WallType.SOLID_BRICK
assert _select_row(session, 1, "old red brick").value is WallType.CAVITY_FILLED
assert _select_row(session, 2, "old red brick").value is WallType.SOLID_BRICK_AS_BUILT_NO_INSULATION_ASSUMED
def test_upsert_persists_unknown_so_a_user_can_resolve_it_later(