Merge branch 'main' of https://github.com/Hestia-Homes/Model into feature/per-cert-mapper-validation

This commit is contained in:
Khalim Conn-Kowlessar 2026-05-30 14:24:10 +00:00
commit a8d6568cbf
17 changed files with 507 additions and 79 deletions

View file

@ -1,13 +1,15 @@
import time
import requests
import pandas as pd import pandas as pd
import numpy as np import numpy as np
from typing import List from typing import Any, List
from functools import lru_cache from functools import lru_cache
from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import MinMaxScaler
from tqdm import tqdm from tqdm import tqdm
from math import sin, cos, sqrt, atan2, radians from math import sin, cos, sqrt, atan2, radians
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
from utils.logger import setup_logger from utils.logger import setup_logger
from recommendations.Costs import Costs from recommendations.Costs import Costs
from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.ml_models.AnnualBillSavings import AnnualBillSavings
@ -57,19 +59,9 @@ class GoogleSolarApi:
# that we calcualte based on the property dimensions, we will correct the roof area # that we calcualte based on the property dimensions, we will correct the roof area
ROOF_AREA_TOLERANCE = 1.25 ROOF_AREA_TOLERANCE = 1.25
# Error Messages def __init__(self, api_key: str, solar_materials: list) -> None:
ENTITY_NOT_FOUND_ERROR = 'Requested entity was not found.'
def __init__(self, api_key, solar_materials: list, max_retries=5):
"""
Initialize the GoogleSolarApi class with the provided API key and maximum retries.
:param api_key: The API key to authenticate requests to the Google Solar API.
:param max_retries: The maximum number of retries for the API request (default is 5).
"""
self.api_key = api_key self.api_key = api_key
self.max_retries = max_retries self._solar_client = GoogleSolarApiClient(api_key)
self.base_url = "https://solar.googleapis.com/v1"
self.insights_data = None self.insights_data = None
self.roof_segments = [] self.roof_segments = []
@ -90,48 +82,11 @@ class GoogleSolarApi:
self.allowed_segment_indices = None self.allowed_segment_indices = None
def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None): def get_building_insights(self, longitude: float, latitude: float, required_quality: str = "MEDIUM") -> dict[str, Any]:
""" try:
Make an API request to retrieve building insights based on the given longitude and latitude, with retry return self._solar_client.get_building_insights(longitude, latitude, required_quality) # type: ignore[arg-type]
mechanism. except BuildingInsightsNotFoundError:
return {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}
:param longitude: The longitude of the location.
:param latitude: The latitude of the location.
:param required_quality: The required quality of the data (default is "MEDIUM").
:param max_retries: The maximum number of retries for the API request (default is None, which uses the
instance's max_retries).
:return: The JSON response containing the building insights data.
"""
if max_retries is None:
max_retries = self.max_retries
insights_url = f"{self.base_url}/buildingInsights:findClosest"
params = {
'location.latitude': f'{latitude:.5f}',
'location.longitude': f'{longitude:.5f}',
'requiredQuality': required_quality,
'key': self.api_key
}
attempt = 0
while attempt < max_retries:
try:
response = requests.get(insights_url, params=params)
response.raise_for_status() # Raise an error for bad status codes
return response.json()
except requests.exceptions.RequestException as e:
if (
(e.response.status_code == 404) &
(e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR)
):
logger.warning("No building insights found for the given location.")
return {"error": self.ENTITY_NOT_FOUND_ERROR}
attempt += 1
print(f"Attempt {attempt} failed: {e}")
time.sleep(2 ** attempt) # Exponential backoff
if attempt >= max_retries:
raise
@lru_cache(maxsize=128) @lru_cache(maxsize=128)
def get( def get(

View file

@ -17,12 +17,14 @@ class HubspotDealData(SQLModel, table=True):
dealstage: Optional[str] = Field(default=None) dealstage: Optional[str] = Field(default=None)
company_id: Optional[str] = Field(default=None) company_id: Optional[str] = Field(default=None)
project_code: Optional[str] = Field(default=None) project_code: Optional[str] = Field(default=None)
project_id: Optional[str] = Field(default=None)
# HubSpot custom properties # HubSpot custom properties
landlord_property_id: Optional[str] = Field(default=None) landlord_property_id: Optional[str] = Field(default=None)
uprn: Optional[str] = Field(default=None) uprn: Optional[str] = Field(default=None)
outcome: Optional[str] = Field(default=None) outcome: Optional[str] = Field(default=None)
outcome_notes: Optional[str] = Field(default=None) outcome_notes: Optional[str] = Field(default=None)
booking_status: Optional[str] = Field(default=None)
major_condition_issue_description: Optional[str] = Field(default=None) major_condition_issue_description: Optional[str] = Field(default=None)
major_condition_issue_photos: Optional[str] = Field(default=None) major_condition_issue_photos: Optional[str] = Field(default=None)
@ -39,7 +41,9 @@ class HubspotDealData(SQLModel, table=True):
damp_mould_and_repairs_comments: Optional[str] = Field(default=None) damp_mould_and_repairs_comments: Optional[str] = Field(default=None)
pre_sap: Optional[str] = Field(default=None) pre_sap: Optional[str] = Field(default=None)
batch: Optional[str] = Field(default=None) batch: Optional[str] = Field(default=None)
batch_description: Optional[str] = Field(default=None)
block_reference: Optional[str] = Field(default=None) block_reference: Optional[str] = Field(default=None)
nonfunded_measures: Optional[str] = Field(default=None)
epc_prn: Optional[str] = Field(default=None) epc_prn: Optional[str] = Field(default=None)
potential_post_sap_score_dropdown: Optional[str] = Field(default=None) potential_post_sap_score_dropdown: Optional[str] = Field(default=None)
ei_score: Optional[str] = Field(default=None) ei_score: Optional[str] = Field(default=None)

View file

@ -0,0 +1,34 @@
import uuid
from sqlmodel import SQLModel, Field, Column, text
from datetime import datetime
from typing import Optional
from sqlalchemy import DateTime
from sqlalchemy.sql import func
class HubspotProjectData(SQLModel, table=True):
__tablename__ = "hubspot_projects_data"
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
project_id: str = Field(index=True, nullable=False, unique=True)
name: Optional[str] = Field(default=None)
created_at: Optional[datetime] = Field(
sa_column=Column(
DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"),
nullable=False,
),
default=func.now(),
)
updated_at: Optional[datetime] = Field(
sa_column=Column(
DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"),
onupdate=func.now(),
nullable=False,
),
default=func.now(),
)

View file

@ -31,6 +31,7 @@ from hubspot.crm.associations.v4.models import ( # type: ignore[reportMissingTy
from backend.app.config import get_settings from backend.app.config import get_settings
from etl.hubspot.company_data import CompanyData from etl.hubspot.company_data import CompanyData
from etl.hubspot.project_data import ProjectData
from utils.logger import setup_logger from utils.logger import setup_logger
import mimetypes import mimetypes
@ -239,6 +240,47 @@ class HubspotClient:
self.logger.info(f"Listing info for deal {deal_id}: {listing_info}") self.logger.info(f"Listing info for deal {deal_id}: {listing_info}")
return listing_info return listing_info
def from_deal_id_get_associated_project(
self, deal_id: str
) -> Optional[ProjectData]:
"""
Get the associated project (custom object "0-970") for a given deal.
Returns a ProjectData with "project_id" and "name", or None if not found.
"""
associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api # type: ignore[reportUnknownMemberType]
projects_api: ObjectsBasicApi = self.client.crm.objects.basic_api # type: ignore[reportUnknownMemberType] # works for custom objects like "project"
response: AssociationsPageResponse = self._call_with_retry(
lambda: associations_api.get_page( # type: ignore[reportUnknownMemberType]
object_type="deals",
object_id=deal_id,
to_object_type="0-970",
limit=1,
)
)
results: list[AssociationsResult] = cast(list[AssociationsResult], response.results) # type: ignore[reportUnknownMemberType]
if not results:
self.logger.info(f"No project association found for deal {deal_id}")
return None
first: AssociationsResult = results[0]
project_id: str = cast(str, first.to_object_id) # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
self.logger.info(f"Associated project ID for deal {deal_id}: {project_id}")
project: HubspotObject = self._call_with_retry(
lambda: projects_api.get_by_id( # type: ignore[reportUnknownMemberType]
object_type="0-970",
object_id=project_id,
properties=["hs_name"],
)
)
project_properties: dict[str, str] = cast(dict[str, str], project.properties) # type: ignore[reportUnknownMemberType]
return ProjectData(
project_id=project_id, name=project_properties.get("hs_name")
)
def from_deal_id_get_info( def from_deal_id_get_info(
self, deal_id: str self, deal_id: str
) -> dict[str, str]: # TODO: add dataclass for this ) -> dict[str, str]: # TODO: add dataclass for this
@ -253,6 +295,7 @@ class HubspotClient:
"pipeline", "pipeline",
"outcome", "outcome",
"outcome_notes", "outcome_notes",
"booking_status",
"project_code", "project_code",
"major_condition_issue_description", "major_condition_issue_description",
"major_condition_issue_photos", "major_condition_issue_photos",
@ -285,7 +328,9 @@ class HubspotClient:
"surveyed_date", "surveyed_date",
"design_type", "design_type",
"batch", "batch",
"batch_description",
"block_reference", "block_reference",
"nonfunded_measures",
"epc_prn", "epc_prn",
"potential_post_sap_score_dropdown", "potential_post_sap_score_dropdown",
"ei_score", "ei_score",
@ -323,17 +368,22 @@ class HubspotClient:
self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}") self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}")
return None return None
def get_deal_and_company_and_listing( def get_deal_and_company_and_listing_and_project(
self, deal_id: str self, deal_id: str
) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]: ) -> tuple[
dict[str, str], Optional[str], Optional[dict[str, str]], Optional[ProjectData]
]:
deal: dict[str, str] = self.from_deal_id_get_info(deal_id) deal: dict[str, str] = self.from_deal_id_get_info(deal_id)
company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id) company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id)
listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing( listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing(
deal_id deal_id
) )
project: Optional[ProjectData] = self.from_deal_id_get_associated_project(
deal_id
)
return deal, company, listing return deal, company, listing, project
def get_company_information(self, company_id: str) -> CompanyData: def get_company_information(self, company_id: str) -> CompanyData:
companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api # type: ignore[reportUnknownMemberType] companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api # type: ignore[reportUnknownMemberType]

View file

@ -4,9 +4,11 @@ from datetime import datetime, timezone
from typing import Dict, Optional from typing import Dict, Optional
from backend.app.db.models.hubspot_deal_data import HubspotDealData from backend.app.db.models.hubspot_deal_data import HubspotDealData
from backend.app.db.models.hubspot_project_data import HubspotProjectData
from backend.app.db.models.hubspot_user import HubspotUser from backend.app.db.models.hubspot_user import HubspotUser
from etl.hubspot.company_data import CompanyData from etl.hubspot.company_data import CompanyData
from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.project_data import ProjectData
from etl.hubspot.s3_uploader import S3Uploader from etl.hubspot.s3_uploader import S3Uploader
from backend.app.db.connection import db_read_session from backend.app.db.connection import db_read_session
from backend.app.db.models.organisation import Organisation from backend.app.db.models.organisation import Organisation
@ -64,6 +66,30 @@ class HubspotDataToDb:
session.commit() session.commit()
return record return record
def upsert_project(self, project: ProjectData) -> HubspotProjectData:
"""Upserts a project record. Updates if project_id exists, otherwise creates new."""
with db_read_session() as session:
project_id = project["project_id"]
name = project.get("name")
existing = session.exec(
select(HubspotProjectData).where(
HubspotProjectData.project_id == project_id
)
).first()
if existing:
existing.name = name
existing.updated_at = datetime.now(timezone.utc)
session.add(existing)
record = existing
else:
record = HubspotProjectData(project_id=project_id, name=name)
session.add(record)
session.commit()
return record
def find_all_deals_with_company_id(self, company_id: str): def find_all_deals_with_company_id(self, company_id: str):
"""Returns a list of deals for a given company_id.""" """Returns a list of deals for a given company_id."""
with db_read_session() as session: with db_read_session() as session:
@ -87,6 +113,7 @@ class HubspotDataToDb:
company: Optional[str], company: Optional[str],
listing: Optional[dict[str, str]], listing: Optional[dict[str, str]],
hubspot_client: HubspotClient, hubspot_client: HubspotClient,
project: Optional[ProjectData] = None,
): ):
""" """
Inserts or updates a deal record. Inserts or updates a deal record.
@ -111,7 +138,9 @@ class HubspotDataToDb:
self._handle_existing_photo_upload(existing, hubspot_client) self._handle_existing_photo_upload(existing, hubspot_client)
print(f"🔄 Updating existing deal (deal_id={deal_id})") print(f"🔄 Updating existing deal (deal_id={deal_id})")
self._update_existing_deal(existing, deal_data, listing, company) self._update_existing_deal(
existing, deal_data, listing, company, project
)
session.add(existing) session.add(existing)
session.commit() session.commit()
@ -121,7 +150,7 @@ class HubspotDataToDb:
else: else:
print(f"🆕 Inserting new deal (deal_id={deal_id})") print(f"🆕 Inserting new deal (deal_id={deal_id})")
new_record: HubspotDealData = self._build_new_deal( new_record: HubspotDealData = self._build_new_deal(
deal_id, deal_data, listing, company deal_id, deal_data, listing, company, project
) )
# Handle upload at insert time # Handle upload at insert time
@ -170,10 +199,12 @@ class HubspotDataToDb:
deal_data: Dict[str, str], deal_data: Dict[str, str],
listing: Optional[dict[str, str]], listing: Optional[dict[str, str]],
company: Optional[str], company: Optional[str],
project: Optional[ProjectData] = None,
): ):
for attr, value in { for attr, value in {
"dealname": deal_data.get("dealname"), "dealname": deal_data.get("dealname"),
"dealstage": deal_data.get("dealstage"), "dealstage": deal_data.get("dealstage"),
"project_id": project["project_id"] if project else None,
"listing_id": listing.get("listing_id", None) if listing else None, "listing_id": listing.get("listing_id", None) if listing else None,
"landlord_property_id": ( "landlord_property_id": (
listing.get("owner_property_id", None) if listing else None listing.get("owner_property_id", None) if listing else None
@ -181,6 +212,7 @@ class HubspotDataToDb:
"uprn": listing.get("national_uprn", None) if listing else None, "uprn": listing.get("national_uprn", None) if listing else None,
"outcome": deal_data.get("outcome"), "outcome": deal_data.get("outcome"),
"outcome_notes": deal_data.get("outcome_notes"), "outcome_notes": deal_data.get("outcome_notes"),
"booking_status": deal_data.get("booking_status"),
"project_code": deal_data.get("project_code"), "project_code": deal_data.get("project_code"),
"company_id": company, "company_id": company,
"major_condition_issue_description": deal_data.get( "major_condition_issue_description": deal_data.get(
@ -200,7 +232,9 @@ class HubspotDataToDb:
), ),
"pre_sap": deal_data.get("pre_sap_score_dropdown"), "pre_sap": deal_data.get("pre_sap_score_dropdown"),
"batch": deal_data.get("batch"), "batch": deal_data.get("batch"),
"batch_description": deal_data.get("batch_description"),
"block_reference": deal_data.get("block_reference"), "block_reference": deal_data.get("block_reference"),
"nonfunded_measures": deal_data.get("nonfunded_measures"),
"epc_prn": deal_data.get("epc_prn"), "epc_prn": deal_data.get("epc_prn"),
"potential_post_sap_score_dropdown": deal_data.get( "potential_post_sap_score_dropdown": deal_data.get(
"potential_post_sap_score_dropdown" "potential_post_sap_score_dropdown"
@ -268,11 +302,13 @@ class HubspotDataToDb:
deal_data: Dict[str, str], deal_data: Dict[str, str],
listing: Optional[dict[str, str]], listing: Optional[dict[str, str]],
company: Optional[str], company: Optional[str],
project: Optional[ProjectData] = None,
) -> HubspotDealData: ) -> HubspotDealData:
return HubspotDealData( return HubspotDealData(
deal_id=deal_id, deal_id=deal_id,
dealname=deal_data.get("dealname"), dealname=deal_data.get("dealname"),
dealstage=deal_data.get("dealstage"), dealstage=deal_data.get("dealstage"),
project_id=project["project_id"] if project else None,
listing_id=listing.get("listing_id") if listing else None, listing_id=listing.get("listing_id") if listing else None,
landlord_property_id=( landlord_property_id=(
listing.get("owner_property_id") if listing else None listing.get("owner_property_id") if listing else None
@ -280,6 +316,7 @@ class HubspotDataToDb:
uprn=listing.get("national_uprn") if listing else None, uprn=listing.get("national_uprn") if listing else None,
outcome=deal_data.get("outcome"), outcome=deal_data.get("outcome"),
outcome_notes=deal_data.get("outcome_notes"), outcome_notes=deal_data.get("outcome_notes"),
booking_status=deal_data.get("booking_status"),
project_code=deal_data.get("project_code"), project_code=deal_data.get("project_code"),
company_id=company, company_id=company,
major_condition_issue_description=deal_data.get( major_condition_issue_description=deal_data.get(
@ -297,7 +334,9 @@ class HubspotDataToDb:
), ),
pre_sap=deal_data.get("pre_sap_score_dropdown"), pre_sap=deal_data.get("pre_sap_score_dropdown"),
batch=deal_data.get("batch"), batch=deal_data.get("batch"),
batch_description=deal_data.get("batch_description"),
block_reference=deal_data.get("block_reference"), block_reference=deal_data.get("block_reference"),
nonfunded_measures=deal_data.get("nonfunded_measures"),
epc_prn=deal_data.get("epc_prn"), epc_prn=deal_data.get("epc_prn"),
potential_post_sap_score_dropdown=deal_data.get( potential_post_sap_score_dropdown=deal_data.get(
"potential_post_sap_score_dropdown" "potential_post_sap_score_dropdown"

View file

@ -53,6 +53,7 @@ class HubspotDealDiffer:
"dealname": "dealname", "dealname": "dealname",
"project_code": "project_code", "project_code": "project_code",
"outcome_notes": "outcome_notes", "outcome_notes": "outcome_notes",
"booking_status": "booking_status",
"major_condition_issue_description": "major_condition_issue_description", "major_condition_issue_description": "major_condition_issue_description",
"major_condition_issue_photos": "major_condition_issue_photos", "major_condition_issue_photos": "major_condition_issue_photos",
"coordination_status__stage_1_": "coordination_status", "coordination_status__stage_1_": "coordination_status",
@ -64,7 +65,9 @@ class HubspotDealDiffer:
"damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments", "damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments",
"pre_sap_score_dropdown": "pre_sap", "pre_sap_score_dropdown": "pre_sap",
"batch": "batch", "batch": "batch",
"batch_description": "batch_description",
"block_reference": "block_reference", "block_reference": "block_reference",
"nonfunded_measures": "nonfunded_measures",
"epc_prn": "epc_prn", "epc_prn": "epc_prn",
"potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown",
"ei_score": "ei_score", "ei_score": "ei_score",

View file

@ -0,0 +1,6 @@
from typing import Optional, TypedDict
class ProjectData(TypedDict):
project_id: str
name: Optional[str]

View file

@ -14,7 +14,7 @@ payload = {
{ {
"task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917", "task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917",
"sub_task_id": "8673913b-1a88-42d7-8578-0449123d94b0", "sub_task_id": "8673913b-1a88-42d7-8578-0449123d94b0",
"hubspot_deal_id": "254427203793", "hubspot_deal_id": "467396027619",
} }
) )
} }

View file

@ -5,6 +5,7 @@ from typing import Any, Dict, Optional
from backend.app.config import get_settings from backend.app.config import get_settings
from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
from etl.hubspot.project_data import ProjectData
from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import (
HubspotTriggerOrchestratorTriggerRequest, HubspotTriggerOrchestratorTriggerRequest,
@ -34,9 +35,10 @@ def handler(body: dict[str, Any], context: Any) -> None:
hubspot_deal: Dict[str, str] hubspot_deal: Dict[str, str]
company: Optional[str] company: Optional[str]
listing: Optional[dict[str, str]] listing: Optional[dict[str, str]]
project: Optional[ProjectData]
hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing( hubspot_deal, company, listing, project = (
hubspot_deal_id hubspot_client.get_deal_and_company_and_listing_and_project(hubspot_deal_id)
) )
deal_changed = False deal_changed = False
@ -47,7 +49,10 @@ def handler(body: dict[str, Any], context: Any) -> None:
db_client: HubspotDataToDb = HubspotDataToDb() db_client: HubspotDataToDb = HubspotDataToDb()
db_client.upsert_organisation(company_data) db_client.upsert_organisation(company_data)
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) if project:
db_client.upsert_project(project)
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client, project)
# ============================== # ==============================
# Orchestration of other lambdas # Orchestration of other lambdas
@ -77,11 +82,15 @@ def handler(body: dict[str, Any], context: Any) -> None:
logger.info( logger.info(
f"Deal {hubspot_deal_id} has been changed, updating database..." f"Deal {hubspot_deal_id} has been changed, updating database..."
) )
if project:
db_client.upsert_project(project)
db_client.upsert_deal( db_client.upsert_deal(
deal_data=hubspot_deal, deal_data=hubspot_deal,
company=company, company=company,
listing=listing, listing=listing,
hubspot_client=hubspot_client, hubspot_client=hubspot_client,
project=project,
) )
deal_changed = True deal_changed = True

View file

@ -71,7 +71,9 @@ class TestHubspotClientIntegration:
def test_get_deal_info_for_db(self, client: HubspotClient): def test_get_deal_info_for_db(self, client: HubspotClient):
deal_id: str = "263490768079" deal_id: str = "263490768079"
deal, company, listing = client.get_deal_and_company_and_listing(deal_id) deal, company, listing, project = (
client.get_deal_and_company_and_listing_and_project(deal_id)
)
assert "dealname" in deal assert "dealname" in deal
assert "dealstage" in deal assert "dealstage" in deal
@ -81,6 +83,8 @@ class TestHubspotClientIntegration:
assert listing is None or "hs_object_id" in listing assert listing is None or "hs_object_id" in listing
assert project is None or "project_id" in project
def test_get_company_information(self, client: HubspotClient): def test_get_company_information(self, client: HubspotClient):
company_id: str = Companies.APPLE.value company_id: str = Companies.APPLE.value

View file

@ -1,4 +1,5 @@
from etl.hubspot.hubspotDataTodB import HubspotDataToDb from etl.hubspot.hubspotDataTodB import HubspotDataToDb
from etl.hubspot.project_data import ProjectData
from backend.app.db.models.hubspot_deal_data import HubspotDealData from backend.app.db.models.hubspot_deal_data import HubspotDealData
@ -32,3 +33,55 @@ def test_update_existing_deal__designer_set__overwrites_existing() -> None:
) )
assert existing.designer == "New Designer" assert existing.designer == "New Designer"
def test_build_new_deal__project_sets_project_id() -> None:
new_deal = _make_instance()._build_new_deal(
deal_id="MOCK_DEAL_ID",
deal_data={},
listing=None,
company=None,
project=ProjectData(project_id="proj-1", name="Project One"),
)
assert new_deal.project_id == "proj-1"
def test_build_new_deal__no_project__project_id_none() -> None:
new_deal = _make_instance()._build_new_deal(
deal_id="MOCK_DEAL_ID",
deal_data={},
listing=None,
company=None,
project=None,
)
assert new_deal.project_id is None
def test_update_existing_deal__project_sets_project_id() -> None:
existing = HubspotDealData(deal_id="MOCK_DEAL_ID")
_make_instance()._update_existing_deal(
existing=existing,
deal_data={},
listing=None,
company=None,
project=ProjectData(project_id="proj-1", name="Project One"),
)
assert existing.project_id == "proj-1"
def test_update_existing_deal__no_project__clears_project_id() -> None:
existing = HubspotDealData(deal_id="MOCK_DEAL_ID", project_id="old-proj")
_make_instance()._update_existing_deal(
existing=existing,
deal_data={},
listing=None,
company=None,
project=None,
)
assert existing.project_id is None

View file

@ -12,6 +12,7 @@ DEAL_ID = "999"
PASHUB_LINK = "https://pashub.example.com/deal/999" PASHUB_LINK = "https://pashub.example.com/deal/999"
MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev"
PASHUB_QUEUE_URL = "https://sqs.test/pashub" PASHUB_QUEUE_URL = "https://sqs.test/pashub"
PROJECT = {"project_id": "proj-1", "name": "Project One"}
def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]:
@ -35,7 +36,8 @@ def run_handler(
hubspot_deal: Dict[str, Any], hubspot_deal: Dict[str, Any],
db_deal: Optional[HubspotDealData], db_deal: Optional[HubspotDealData],
listing: Optional[dict], listing: Optional[dict],
) -> MagicMock: project: Optional[dict] = None,
) -> tuple[MagicMock, MagicMock]:
mock_sqs = MagicMock() mock_sqs = MagicMock()
mock_sqs.send_message.return_value = {"MessageId": "test-id"} mock_sqs.send_message.return_value = {"MessageId": "test-id"}
@ -47,10 +49,12 @@ def run_handler(
): ):
mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal
mock_db_cls.return_value.upsert_deal.return_value = None mock_db_cls.return_value.upsert_deal.return_value = None
mock_hs_cls.return_value.get_deal_and_company_and_listing.return_value = ( mock_hs = mock_hs_cls.return_value
mock_hs.get_deal_and_company_and_listing_and_project.return_value = (
hubspot_deal, hubspot_deal,
None, None,
listing, listing,
project,
) )
mock_boto3.client.return_value = mock_sqs mock_boto3.client.return_value = mock_sqs
mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL
@ -58,7 +62,7 @@ def run_handler(
handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "")
return mock_sqs return mock_sqs, mock_db_cls.return_value
# ==================================== # ====================================
@ -72,7 +76,7 @@ def test_new_deal_surveyed__sends_magicplan_sqs() -> None:
listing = {"national_uprn": UPRN} listing = {"national_uprn": UPRN}
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing)
# Assert # Assert
mock_sqs.send_message.assert_called_once_with( mock_sqs.send_message.assert_called_once_with(
@ -88,7 +92,7 @@ def test_new_deal_not_surveyed__no_magicplan_sqs() -> None:
hubspot_deal = make_hubspot_deal(outcome="assessed") hubspot_deal = make_hubspot_deal(outcome="assessed")
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_not_called() mock_sqs.send_message.assert_not_called()
@ -99,7 +103,7 @@ def test_new_deal_surveyed_no_listing__magicplan_sqs_uprn_is_null() -> None:
hubspot_deal = make_hubspot_deal(outcome="surveyed") hubspot_deal = make_hubspot_deal(outcome="surveyed")
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_called_once_with( mock_sqs.send_message.assert_called_once_with(
@ -122,7 +126,7 @@ def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None:
listing = {"national_uprn": UPRN} listing = {"national_uprn": UPRN}
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing)
# Assert # Assert
mock_sqs.send_message.assert_called_once_with( mock_sqs.send_message.assert_called_once_with(
@ -139,7 +143,7 @@ def test_existing_deal_already_surveyed__no_magicplan_sqs() -> None:
hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name")
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_not_called() mock_sqs.send_message.assert_not_called()
@ -155,7 +159,7 @@ def test_new_deal_with_pashub_link__sends_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK)
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_called_once_with( mock_sqs.send_message.assert_called_once_with(
@ -179,7 +183,7 @@ def test_new_deal_no_pashub_link__no_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal() hubspot_deal = make_hubspot_deal()
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_not_called() mock_sqs.send_message.assert_not_called()
@ -196,7 +200,7 @@ def test_existing_deal_pashub_link_added__sends_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK)
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_called_once_with( mock_sqs.send_message.assert_called_once_with(
@ -221,7 +225,67 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None:
hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name") hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name")
# Act # Act
mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None)
# Assert # Assert
mock_sqs.send_message.assert_not_called() mock_sqs.send_message.assert_not_called()
# ====================================
# PROJECT upsert
# ====================================
def test_new_deal_with_project__upserts_project() -> None:
# Arrange
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_called_once_with(PROJECT)
def test_new_deal_no_project__no_project_upsert() -> None:
# Arrange
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=None
)
# Assert
mock_db.upsert_project.assert_not_called()
def test_existing_deal_changed_with_project__upserts_project() -> None:
# Arrange
db_deal = make_db_deal(outcome="assessed")
hubspot_deal = make_hubspot_deal(outcome="surveyed")
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_called_once_with(PROJECT)
def test_existing_deal_unchanged__no_project_upsert() -> None:
# Arrange: db deal matches hubspot deal, so the differ reports no change
db_deal = make_db_deal(dealname=DEAL_NAME)
hubspot_deal = make_hubspot_deal()
# Act
_, mock_db = run_handler(
hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT
)
# Assert
mock_db.upsert_project.assert_not_called()
mock_db.upsert_deal.assert_not_called()

View file

View file

@ -0,0 +1,50 @@
import time
from typing import Any, Literal, Optional
import requests
class BuildingInsightsNotFoundError(Exception):
pass
class GoogleSolarApiClient:
base_url: str = "https://solar.googleapis.com/v1"
MAX_RETRIES: int = 5
ENTITY_NOT_FOUND_ERROR: str = "Requested entity was not found."
def __init__(self, api_key: str) -> None:
self._api_key = api_key
def get_building_insights(
self,
longitude: float,
latitude: float,
required_quality: Literal["HIGH", "MEDIUM", "LOW"] = "MEDIUM",
) -> dict[str, Any]:
insights_url = f"{self.base_url}/buildingInsights:findClosest"
params: dict[str, str] = {
"location.latitude": f"{latitude:.5f}",
"location.longitude": f"{longitude:.5f}",
"requiredQuality": required_quality,
"key": self._api_key,
}
last_exc: Optional[Exception] = None
for attempt in range(self.MAX_RETRIES):
try:
response = requests.get(insights_url, params=params)
response.raise_for_status()
result: dict[str, Any] = response.json()
return result
except requests.exceptions.RequestException as e:
if (
e.response is not None
and e.response.status_code == 404
and e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR
):
raise BuildingInsightsNotFoundError() from e
last_exc = e
time.sleep(2 ** attempt)
assert last_exc is not None
raise last_exc

View file

View file

@ -0,0 +1,110 @@
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
import requests
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
# ---------------------------------------------------------------------------
# Slice 1: Successful response returns parsed JSON
# ---------------------------------------------------------------------------
def test_get_building_insights_returns_parsed_json() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
mock_response = MagicMock()
mock_response.json.return_value = payload
with patch("requests.get", return_value=mock_response) as mock_get:
mock_response.raise_for_status.return_value = None
# Act
result = client.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
mock_get.assert_called_once()
# ---------------------------------------------------------------------------
# Slice 2: Transient HTTP errors trigger retries
# ---------------------------------------------------------------------------
def test_get_building_insights_retries_on_transient_error() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
transient_error = requests.exceptions.ConnectionError("timeout")
transient_error.response = None # type: ignore[attr-defined]
success_response = MagicMock()
success_response.json.return_value = payload
success_response.raise_for_status.return_value = None
with patch("requests.get", side_effect=[transient_error, success_response]) as mock_get:
with patch("time.sleep"):
# Act
result = client.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
assert mock_get.call_count == 2
# ---------------------------------------------------------------------------
# Slice 3: 404 + entity-not-found raises BuildingInsightsNotFoundError
# ---------------------------------------------------------------------------
def test_get_building_insights_raises_on_entity_not_found() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
not_found_response = MagicMock()
not_found_response.status_code = 404
not_found_response.json.return_value = {
"error": {"message": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}
}
http_error = requests.exceptions.HTTPError("404")
http_error.response = not_found_response
with patch("requests.get") as mock_get:
mock_get.return_value.raise_for_status.side_effect = http_error
mock_get.return_value.response = not_found_response
# Act / Assert
with pytest.raises(BuildingInsightsNotFoundError):
client.get_building_insights(longitude=-0.1278, latitude=51.5074)
assert mock_get.call_count == 1
# ---------------------------------------------------------------------------
# Slice 4: Retry exhaustion propagates the exception to the caller
# ---------------------------------------------------------------------------
def test_get_building_insights_propagates_exception_after_retry_exhaustion() -> None:
# Arrange
client = GoogleSolarApiClient(api_key="test-key")
error = requests.exceptions.ConnectionError("persistent failure")
error.response = None # type: ignore[attr-defined]
with patch("requests.get", side_effect=error) as mock_get:
with patch("time.sleep"):
# Act / Assert
with pytest.raises(requests.exceptions.ConnectionError):
client.get_building_insights(longitude=-0.1278, latitude=51.5074)
assert mock_get.call_count == GoogleSolarApiClient.MAX_RETRIES

View file

@ -0,0 +1,47 @@
from typing import Any
from unittest.mock import patch
from backend.apis.GoogleSolarApi import GoogleSolarApi
from infrastructure.solar.google_solar_api_client import (
BuildingInsightsNotFoundError,
GoogleSolarApiClient,
)
# ---------------------------------------------------------------------------
# Slice 1: get_building_insights delegates to _solar_client
# ---------------------------------------------------------------------------
def test_get_building_insights_delegates_to_solar_client() -> None:
# Arrange
payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}}
with patch.object(GoogleSolarApiClient, "get_building_insights", return_value=payload):
api = GoogleSolarApi(api_key="test-key", solar_materials=[])
# Act
result = api.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == payload
# ---------------------------------------------------------------------------
# Slice 2: BuildingInsightsNotFoundError translates to sentinel dict
# ---------------------------------------------------------------------------
def test_get_building_insights_returns_sentinel_on_not_found() -> None:
# Arrange
with patch.object(
GoogleSolarApiClient,
"get_building_insights",
side_effect=BuildingInsightsNotFoundError,
):
api = GoogleSolarApi(api_key="test-key", solar_materials=[])
# Act
result = api.get_building_insights(longitude=-0.1278, latitude=51.5074)
# Assert
assert result == {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR}