From 629fc34a0f6f437351ab08a855bff2f82bb0ea83 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:46:47 +0000 Subject: [PATCH 01/13] =?UTF-8?q?GoogleSolarApiClient=20fetches=20building?= =?UTF-8?q?=20insights=20from=20the=20Solar=20API=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/solar/__init__.py | 0 .../solar/google_solar_api_client.py | 22 +++++++++++++ tests/infrastructure/solar/__init__.py | 0 .../solar/test_google_solar_api_client.py | 33 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 infrastructure/solar/__init__.py create mode 100644 infrastructure/solar/google_solar_api_client.py create mode 100644 tests/infrastructure/solar/__init__.py create mode 100644 tests/infrastructure/solar/test_google_solar_api_client.py diff --git a/infrastructure/solar/__init__.py b/infrastructure/solar/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/solar/google_solar_api_client.py b/infrastructure/solar/google_solar_api_client.py new file mode 100644 index 00000000..5e4698e9 --- /dev/null +++ b/infrastructure/solar/google_solar_api_client.py @@ -0,0 +1,22 @@ +from typing import Any, Literal + + +class BuildingInsightsNotFoundError(Exception): + pass + + +class GoogleSolarApiClient: + base_url: str = "https://solar.googleapis.com/v1" + MAX_RETRIES: int = 5 + ENTITY_NOT_FOUND_ERROR: str = "Requested entity was not found." + + def __init__(self, api_key: str) -> None: + raise NotImplementedError + + def get_building_insights( + self, + longitude: float, + latitude: float, + required_quality: Literal["HIGH", "MEDIUM", "LOW"] = "MEDIUM", + ) -> dict[str, Any]: + raise NotImplementedError diff --git a/tests/infrastructure/solar/__init__.py b/tests/infrastructure/solar/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/infrastructure/solar/test_google_solar_api_client.py b/tests/infrastructure/solar/test_google_solar_api_client.py new file mode 100644 index 00000000..b7a7e45d --- /dev/null +++ b/tests/infrastructure/solar/test_google_solar_api_client.py @@ -0,0 +1,33 @@ +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from infrastructure.solar.google_solar_api_client import ( + BuildingInsightsNotFoundError, + GoogleSolarApiClient, +) + + +# --------------------------------------------------------------------------- +# Slice 1: Successful response returns parsed JSON +# --------------------------------------------------------------------------- + + +def test_get_building_insights_returns_parsed_json() -> None: + # Arrange + client = GoogleSolarApiClient(api_key="test-key") + payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}} + mock_response = MagicMock() + mock_response.json.return_value = payload + + with patch("requests.get", return_value=mock_response) as mock_get: + mock_response.raise_for_status.return_value = None + + # Act + result = client.get_building_insights(longitude=-0.1278, latitude=51.5074) + + # Assert + assert result == payload + mock_get.assert_called_once() From b0106fa93def83492d85a93f6cc5a00e3b539fff Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:48:45 +0000 Subject: [PATCH 02/13] =?UTF-8?q?GoogleSolarApiClient=20fetches=20building?= =?UTF-8?q?=20insights=20from=20the=20Solar=20API=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/solar/google_solar_api_client.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/infrastructure/solar/google_solar_api_client.py b/infrastructure/solar/google_solar_api_client.py index 5e4698e9..4edea335 100644 --- a/infrastructure/solar/google_solar_api_client.py +++ b/infrastructure/solar/google_solar_api_client.py @@ -1,5 +1,7 @@ from typing import Any, Literal +import requests + class BuildingInsightsNotFoundError(Exception): pass @@ -11,7 +13,7 @@ class GoogleSolarApiClient: ENTITY_NOT_FOUND_ERROR: str = "Requested entity was not found." def __init__(self, api_key: str) -> None: - raise NotImplementedError + self._api_key = api_key def get_building_insights( self, @@ -19,4 +21,14 @@ class GoogleSolarApiClient: latitude: float, required_quality: Literal["HIGH", "MEDIUM", "LOW"] = "MEDIUM", ) -> dict[str, Any]: - raise NotImplementedError + insights_url = f"{self.base_url}/buildingInsights:findClosest" + params: dict[str, str] = { + "location.latitude": f"{latitude:.5f}", + "location.longitude": f"{longitude:.5f}", + "requiredQuality": required_quality, + "key": self._api_key, + } + response = requests.get(insights_url, params=params) + response.raise_for_status() + result: dict[str, Any] = response.json() + return result From 44217bf36162096629467a7e7a921e07d8c05787 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:49:57 +0000 Subject: [PATCH 03/13] =?UTF-8?q?GoogleSolarApiClient=20retries=20on=20tra?= =?UTF-8?q?nsient=20HTTP=20errors=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solar/test_google_solar_api_client.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/infrastructure/solar/test_google_solar_api_client.py b/tests/infrastructure/solar/test_google_solar_api_client.py index b7a7e45d..9a38e836 100644 --- a/tests/infrastructure/solar/test_google_solar_api_client.py +++ b/tests/infrastructure/solar/test_google_solar_api_client.py @@ -31,3 +31,30 @@ def test_get_building_insights_returns_parsed_json() -> None: # Assert assert result == payload mock_get.assert_called_once() + + +# --------------------------------------------------------------------------- +# Slice 2: Transient HTTP errors trigger retries +# --------------------------------------------------------------------------- + + +def test_get_building_insights_retries_on_transient_error() -> None: + # Arrange + client = GoogleSolarApiClient(api_key="test-key") + payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}} + + transient_error = requests.exceptions.ConnectionError("timeout") + transient_error.response = None # type: ignore[attr-defined] + + success_response = MagicMock() + success_response.json.return_value = payload + success_response.raise_for_status.return_value = None + + with patch("requests.get", side_effect=[transient_error, success_response]) as mock_get: + with patch("time.sleep"): + # Act + result = client.get_building_insights(longitude=-0.1278, latitude=51.5074) + + # Assert + assert result == payload + assert mock_get.call_count == 2 From 497ef1faed822c400dd75fc3ee7f8cc23aeb087e Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:51:01 +0000 Subject: [PATCH 04/13] =?UTF-8?q?GoogleSolarApiClient=20retries=20on=20tra?= =?UTF-8?q?nsient=20HTTP=20errors=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solar/google_solar_api_client.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/infrastructure/solar/google_solar_api_client.py b/infrastructure/solar/google_solar_api_client.py index 4edea335..9112b980 100644 --- a/infrastructure/solar/google_solar_api_client.py +++ b/infrastructure/solar/google_solar_api_client.py @@ -1,4 +1,5 @@ -from typing import Any, Literal +import time +from typing import Any, Literal, Optional import requests @@ -28,7 +29,16 @@ class GoogleSolarApiClient: "requiredQuality": required_quality, "key": self._api_key, } - response = requests.get(insights_url, params=params) - response.raise_for_status() - result: dict[str, Any] = response.json() - return result + last_exc: Optional[Exception] = None + for attempt in range(self.MAX_RETRIES): + try: + response = requests.get(insights_url, params=params) + response.raise_for_status() + result: dict[str, Any] = response.json() + return result + except requests.exceptions.RequestException as e: + last_exc = e + time.sleep(2 ** attempt) + + assert last_exc is not None + raise last_exc From 573656be64416ae2bdae384fd6d03aacce70e466 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:52:21 +0000 Subject: [PATCH 05/13] =?UTF-8?q?GoogleSolarApiClient=20raises=20BuildingI?= =?UTF-8?q?nsightsNotFoundError=20on=20404=20entity-not-found=20?= =?UTF-8?q?=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solar/test_google_solar_api_client.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/infrastructure/solar/test_google_solar_api_client.py b/tests/infrastructure/solar/test_google_solar_api_client.py index 9a38e836..450113a7 100644 --- a/tests/infrastructure/solar/test_google_solar_api_client.py +++ b/tests/infrastructure/solar/test_google_solar_api_client.py @@ -58,3 +58,32 @@ def test_get_building_insights_retries_on_transient_error() -> None: # Assert assert result == payload assert mock_get.call_count == 2 + + +# --------------------------------------------------------------------------- +# Slice 3: 404 + entity-not-found raises BuildingInsightsNotFoundError +# --------------------------------------------------------------------------- + + +def test_get_building_insights_raises_on_entity_not_found() -> None: + # Arrange + client = GoogleSolarApiClient(api_key="test-key") + + not_found_response = MagicMock() + not_found_response.status_code = 404 + not_found_response.json.return_value = { + "error": {"message": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR} + } + + http_error = requests.exceptions.HTTPError("404") + http_error.response = not_found_response + + with patch("requests.get") as mock_get: + mock_get.return_value.raise_for_status.side_effect = http_error + mock_get.return_value.response = not_found_response + + # Act / Assert + with pytest.raises(BuildingInsightsNotFoundError): + client.get_building_insights(longitude=-0.1278, latitude=51.5074) + + assert mock_get.call_count == 1 From d1ca9be5801b4f42297024dc62db97f3b6de3fcc Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:52:52 +0000 Subject: [PATCH 06/13] =?UTF-8?q?GoogleSolarApiClient=20raises=20BuildingI?= =?UTF-8?q?nsightsNotFoundError=20on=20404=20entity-not-found=20?= =?UTF-8?q?=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/solar/google_solar_api_client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infrastructure/solar/google_solar_api_client.py b/infrastructure/solar/google_solar_api_client.py index 9112b980..28e91866 100644 --- a/infrastructure/solar/google_solar_api_client.py +++ b/infrastructure/solar/google_solar_api_client.py @@ -37,6 +37,12 @@ class GoogleSolarApiClient: result: dict[str, Any] = response.json() return result except requests.exceptions.RequestException as e: + if ( + e.response is not None + and e.response.status_code == 404 + and e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR + ): + raise BuildingInsightsNotFoundError() from e last_exc = e time.sleep(2 ** attempt) From b1933c07c351019a45ea337ee12173041eb4f8f7 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:53:39 +0000 Subject: [PATCH 07/13] =?UTF-8?q?GoogleSolarApiClient=20propagates=20excep?= =?UTF-8?q?tion=20after=20retry=20exhaustion=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solar/test_google_solar_api_client.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/infrastructure/solar/test_google_solar_api_client.py b/tests/infrastructure/solar/test_google_solar_api_client.py index 450113a7..d4328fc0 100644 --- a/tests/infrastructure/solar/test_google_solar_api_client.py +++ b/tests/infrastructure/solar/test_google_solar_api_client.py @@ -87,3 +87,24 @@ def test_get_building_insights_raises_on_entity_not_found() -> None: client.get_building_insights(longitude=-0.1278, latitude=51.5074) assert mock_get.call_count == 1 + + +# --------------------------------------------------------------------------- +# Slice 4: Retry exhaustion propagates the exception to the caller +# --------------------------------------------------------------------------- + + +def test_get_building_insights_propagates_exception_after_retry_exhaustion() -> None: + # Arrange + client = GoogleSolarApiClient(api_key="test-key") + + error = requests.exceptions.ConnectionError("persistent failure") + error.response = None # type: ignore[attr-defined] + + with patch("requests.get", side_effect=error) as mock_get: + with patch("time.sleep"): + # Act / Assert + with pytest.raises(requests.exceptions.ConnectionError): + client.get_building_insights(longitude=-0.1278, latitude=51.5074) + + assert mock_get.call_count == GoogleSolarApiClient.MAX_RETRIES From 521294ad917e663f3378652d55235ab3f274b202 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 15:59:22 +0000 Subject: [PATCH 08/13] =?UTF-8?q?GoogleSolarApi=20delegates=20get=5Fbuildi?= =?UTF-8?q?ng=5Finsights=20to=20GoogleSolarApiClient=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../solar/test_google_solar_api_wiring.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/infrastructure/solar/test_google_solar_api_wiring.py diff --git a/tests/infrastructure/solar/test_google_solar_api_wiring.py b/tests/infrastructure/solar/test_google_solar_api_wiring.py new file mode 100644 index 00000000..52618f9d --- /dev/null +++ b/tests/infrastructure/solar/test_google_solar_api_wiring.py @@ -0,0 +1,26 @@ +from typing import Any +from unittest.mock import MagicMock, patch + +from backend.apis.GoogleSolarApi import GoogleSolarApi +from infrastructure.solar.google_solar_api_client import ( + BuildingInsightsNotFoundError, + GoogleSolarApiClient, +) + + +# --------------------------------------------------------------------------- +# Slice 1: get_building_insights delegates to _solar_client +# --------------------------------------------------------------------------- + + +def test_get_building_insights_delegates_to_solar_client() -> None: + # Arrange + payload: dict[str, Any] = {"solarPotential": {"maxArrayPanelsCount": 20}} + with patch.object(GoogleSolarApiClient, "get_building_insights", return_value=payload): + api = GoogleSolarApi(api_key="test-key", solar_materials=[]) + + # Act + result = api.get_building_insights(longitude=-0.1278, latitude=51.5074) + + # Assert + assert result == payload From 50e9d887527ce147164d6f8007b43cf3d6a56d15 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 16:00:59 +0000 Subject: [PATCH 09/13] =?UTF-8?q?GoogleSolarApi=20delegates=20get=5Fbuildi?= =?UTF-8?q?ng=5Finsights=20to=20GoogleSolarApiClient=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/apis/GoogleSolarApi.py | 65 ++++++---------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 6fc5daa6..589d3a04 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -8,6 +8,10 @@ from sklearn.preprocessing import MinMaxScaler from tqdm import tqdm from math import sin, cos, sqrt, atan2, radians +from infrastructure.solar.google_solar_api_client import ( + BuildingInsightsNotFoundError, + GoogleSolarApiClient, +) from utils.logger import setup_logger from recommendations.Costs import Costs from backend.ml_models.AnnualBillSavings import AnnualBillSavings @@ -57,19 +61,9 @@ class GoogleSolarApi: # that we calcualte based on the property dimensions, we will correct the roof area ROOF_AREA_TOLERANCE = 1.25 - # Error Messages - ENTITY_NOT_FOUND_ERROR = 'Requested entity was not found.' - - def __init__(self, api_key, solar_materials: list, max_retries=5): - """ - Initialize the GoogleSolarApi class with the provided API key and maximum retries. - - :param api_key: The API key to authenticate requests to the Google Solar API. - :param max_retries: The maximum number of retries for the API request (default is 5). - """ + def __init__(self, api_key: str, solar_materials: list) -> None: self.api_key = api_key - self.max_retries = max_retries - self.base_url = "https://solar.googleapis.com/v1" + self._solar_client = GoogleSolarApiClient(api_key) self.insights_data = None self.roof_segments = [] @@ -90,48 +84,11 @@ class GoogleSolarApi: self.allowed_segment_indices = None - def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None): - """ - Make an API request to retrieve building insights based on the given longitude and latitude, with retry - mechanism. - - :param longitude: The longitude of the location. - :param latitude: The latitude of the location. - :param required_quality: The required quality of the data (default is "MEDIUM"). - :param max_retries: The maximum number of retries for the API request (default is None, which uses the - instance's max_retries). - :return: The JSON response containing the building insights data. - """ - if max_retries is None: - max_retries = self.max_retries - - insights_url = f"{self.base_url}/buildingInsights:findClosest" - params = { - 'location.latitude': f'{latitude:.5f}', - 'location.longitude': f'{longitude:.5f}', - 'requiredQuality': required_quality, - 'key': self.api_key - } - - attempt = 0 - while attempt < max_retries: - try: - response = requests.get(insights_url, params=params) - response.raise_for_status() # Raise an error for bad status codes - return response.json() - except requests.exceptions.RequestException as e: - if ( - (e.response.status_code == 404) & - (e.response.json()["error"]["message"] == self.ENTITY_NOT_FOUND_ERROR) - ): - logger.warning("No building insights found for the given location.") - return {"error": self.ENTITY_NOT_FOUND_ERROR} - - attempt += 1 - print(f"Attempt {attempt} failed: {e}") - time.sleep(2 ** attempt) # Exponential backoff - if attempt >= max_retries: - raise + def get_building_insights(self, longitude: float, latitude: float, required_quality: str = "MEDIUM") -> dict: + try: + return self._solar_client.get_building_insights(longitude, latitude, required_quality) # type: ignore[arg-type] + except BuildingInsightsNotFoundError: + return {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR} @lru_cache(maxsize=128) def get( From 0d4462d1319eb54c3fde064806d2f473bd2aeef8 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 21 May 2026 16:06:54 +0000 Subject: [PATCH 10/13] =?UTF-8?q?GoogleSolarApi=20translates=20BuildingIns?= =?UTF-8?q?ightsNotFoundError=20to=20sentinel=20dict=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/apis/GoogleSolarApi.py | 6 ++--- .../solar/test_google_solar_api_wiring.py | 23 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 589d3a04..7c90307c 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -1,8 +1,6 @@ -import time -import requests import pandas as pd import numpy as np -from typing import List +from typing import Any, List from functools import lru_cache from sklearn.preprocessing import MinMaxScaler from tqdm import tqdm @@ -84,7 +82,7 @@ class GoogleSolarApi: self.allowed_segment_indices = None - def get_building_insights(self, longitude: float, latitude: float, required_quality: str = "MEDIUM") -> dict: + def get_building_insights(self, longitude: float, latitude: float, required_quality: str = "MEDIUM") -> dict[str, Any]: try: return self._solar_client.get_building_insights(longitude, latitude, required_quality) # type: ignore[arg-type] except BuildingInsightsNotFoundError: diff --git a/tests/infrastructure/solar/test_google_solar_api_wiring.py b/tests/infrastructure/solar/test_google_solar_api_wiring.py index 52618f9d..77988645 100644 --- a/tests/infrastructure/solar/test_google_solar_api_wiring.py +++ b/tests/infrastructure/solar/test_google_solar_api_wiring.py @@ -1,5 +1,5 @@ from typing import Any -from unittest.mock import MagicMock, patch +from unittest.mock import patch from backend.apis.GoogleSolarApi import GoogleSolarApi from infrastructure.solar.google_solar_api_client import ( @@ -24,3 +24,24 @@ def test_get_building_insights_delegates_to_solar_client() -> None: # Assert assert result == payload + + +# --------------------------------------------------------------------------- +# Slice 2: BuildingInsightsNotFoundError translates to sentinel dict +# --------------------------------------------------------------------------- + + +def test_get_building_insights_returns_sentinel_on_not_found() -> None: + # Arrange + with patch.object( + GoogleSolarApiClient, + "get_building_insights", + side_effect=BuildingInsightsNotFoundError, + ): + api = GoogleSolarApi(api_key="test-key", solar_materials=[]) + + # Act + result = api.get_building_insights(longitude=-0.1278, latitude=51.5074) + + # Assert + assert result == {"error": GoogleSolarApiClient.ENTITY_NOT_FOUND_ERROR} From 0a99c92eb41e0adece18de663af22f883222897a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 27 May 2026 10:30:19 +0000 Subject: [PATCH 11/13] added batch description and nonfunded measures --- backend/app/db/models/hubspot_deal_data.py | 2 ++ etl/hubspot/hubspotClient.py | 2 ++ etl/hubspot/hubspotDataTodB.py | 4 ++++ etl/hubspot/hubspot_deal_differ.py | 2 ++ 4 files changed, 10 insertions(+) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 0ee58d54..dd5cdb14 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -39,7 +39,9 @@ class HubspotDealData(SQLModel, table=True): damp_mould_and_repairs_comments: Optional[str] = Field(default=None) pre_sap: Optional[str] = Field(default=None) batch: Optional[str] = Field(default=None) + batch_description: Optional[str] = Field(default=None) block_reference: Optional[str] = Field(default=None) + nonfunded_measures: Optional[str] = Field(default=None) epc_prn: Optional[str] = Field(default=None) potential_post_sap_score_dropdown: Optional[str] = Field(default=None) ei_score: Optional[str] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 4c9cb1e6..769b8ea6 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -285,7 +285,9 @@ class HubspotClient: "surveyed_date", "design_type", "batch", + "batch_description", "block_reference", + "nonfunded_measures", "epc_prn", "potential_post_sap_score_dropdown", "ei_score", diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index b160d563..a6e19ef4 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -200,7 +200,9 @@ class HubspotDataToDb: ), "pre_sap": deal_data.get("pre_sap_score_dropdown"), "batch": deal_data.get("batch"), + "batch_description": deal_data.get("batch_description"), "block_reference": deal_data.get("block_reference"), + "nonfunded_measures": deal_data.get("nonfunded_measures"), "epc_prn": deal_data.get("epc_prn"), "potential_post_sap_score_dropdown": deal_data.get( "potential_post_sap_score_dropdown" @@ -297,7 +299,9 @@ class HubspotDataToDb: ), pre_sap=deal_data.get("pre_sap_score_dropdown"), batch=deal_data.get("batch"), + batch_description=deal_data.get("batch_description"), block_reference=deal_data.get("block_reference"), + nonfunded_measures=deal_data.get("nonfunded_measures"), epc_prn=deal_data.get("epc_prn"), potential_post_sap_score_dropdown=deal_data.get( "potential_post_sap_score_dropdown" diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index da0072c1..dca6e7ea 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -64,7 +64,9 @@ class HubspotDealDiffer: "damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments", "pre_sap_score_dropdown": "pre_sap", "batch": "batch", + "batch_description": "batch_description", "block_reference": "block_reference", + "nonfunded_measures": "nonfunded_measures", "epc_prn": "epc_prn", "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", "ei_score": "ei_score", From 2b29b4b4583316d12553e927627ff31e042817db Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 May 2026 10:54:15 +0000 Subject: [PATCH 12/13] hubspot projects data is scraped --- backend/app/db/models/hubspot_deal_data.py | 1 + backend/app/db/models/hubspot_project_data.py | 34 +++++++ etl/hubspot/hubspotClient.py | 53 ++++++++++- etl/hubspot/hubspotDataTodB.py | 37 +++++++- etl/hubspot/project_data.py | 6 ++ .../local_handler/invoke_local_lambda.py | 2 +- etl/hubspot/scripts/scraper/main.py | 15 +++- .../tests/test_hubspot_client_integration.py | 6 +- etl/hubspot/tests/test_hubspot_data_to_db.py | 53 +++++++++++ etl/hubspot/tests/test_scraper_handler.py | 88 ++++++++++++++++--- 10 files changed, 273 insertions(+), 22 deletions(-) create mode 100644 backend/app/db/models/hubspot_project_data.py create mode 100644 etl/hubspot/project_data.py diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index dd5cdb14..b0876c03 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -17,6 +17,7 @@ class HubspotDealData(SQLModel, table=True): dealstage: Optional[str] = Field(default=None) company_id: Optional[str] = Field(default=None) project_code: Optional[str] = Field(default=None) + project_id: Optional[str] = Field(default=None) # HubSpot custom properties landlord_property_id: Optional[str] = Field(default=None) diff --git a/backend/app/db/models/hubspot_project_data.py b/backend/app/db/models/hubspot_project_data.py new file mode 100644 index 00000000..5d5df783 --- /dev/null +++ b/backend/app/db/models/hubspot_project_data.py @@ -0,0 +1,34 @@ +import uuid +from sqlmodel import SQLModel, Field, Column, text +from datetime import datetime +from typing import Optional +from sqlalchemy import DateTime +from sqlalchemy.sql import func + + +class HubspotProjectData(SQLModel, table=True): + __tablename__ = "hubspot_projects_data" + + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + + project_id: str = Field(index=True, nullable=False, unique=True) + name: Optional[str] = Field(default=None) + + created_at: Optional[datetime] = Field( + sa_column=Column( + DateTime(timezone=True), + server_default=text("(NOW() AT TIME ZONE 'utc')"), + nullable=False, + ), + default=func.now(), + ) + + updated_at: Optional[datetime] = Field( + sa_column=Column( + DateTime(timezone=True), + server_default=text("(NOW() AT TIME ZONE 'utc')"), + onupdate=func.now(), + nullable=False, + ), + default=func.now(), + ) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 769b8ea6..4c542b40 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -31,6 +31,7 @@ from hubspot.crm.associations.v4.models import ( # type: ignore[reportMissingTy from backend.app.config import get_settings from etl.hubspot.company_data import CompanyData +from etl.hubspot.project_data import ProjectData from utils.logger import setup_logger import mimetypes @@ -239,6 +240,47 @@ class HubspotClient: self.logger.info(f"Listing info for deal {deal_id}: {listing_info}") return listing_info + def from_deal_id_get_associated_project( + self, deal_id: str + ) -> Optional[ProjectData]: + """ + Get the associated project (custom object "0-970") for a given deal. + Returns a ProjectData with "project_id" and "name", or None if not found. + """ + associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api # type: ignore[reportUnknownMemberType] + projects_api: ObjectsBasicApi = self.client.crm.objects.basic_api # type: ignore[reportUnknownMemberType] # works for custom objects like "project" + + response: AssociationsPageResponse = self._call_with_retry( + lambda: associations_api.get_page( # type: ignore[reportUnknownMemberType] + object_type="deals", + object_id=deal_id, + to_object_type="0-970", + limit=1, + ) + ) + + results: list[AssociationsResult] = cast(list[AssociationsResult], response.results) # type: ignore[reportUnknownMemberType] + if not results: + self.logger.info(f"No project association found for deal {deal_id}") + return None + + first: AssociationsResult = results[0] + project_id: str = cast(str, first.to_object_id) # type: ignore[reportUnknownMemberType, reportUnknownVariableType] + self.logger.info(f"Associated project ID for deal {deal_id}: {project_id}") + + project: HubspotObject = self._call_with_retry( + lambda: projects_api.get_by_id( # type: ignore[reportUnknownMemberType] + object_type="0-970", + object_id=project_id, + properties=["hs_name"], + ) + ) + + project_properties: dict[str, str] = cast(dict[str, str], project.properties) # type: ignore[reportUnknownMemberType] + return ProjectData( + project_id=project_id, name=project_properties.get("hs_name") + ) + def from_deal_id_get_info( self, deal_id: str ) -> dict[str, str]: # TODO: add dataclass for this @@ -325,17 +367,22 @@ class HubspotClient: self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}") return None - def get_deal_and_company_and_listing( + def get_deal_and_company_and_listing_and_project( self, deal_id: str - ) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]: + ) -> tuple[ + dict[str, str], Optional[str], Optional[dict[str, str]], Optional[ProjectData] + ]: deal: dict[str, str] = self.from_deal_id_get_info(deal_id) company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id) listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing( deal_id ) + project: Optional[ProjectData] = self.from_deal_id_get_associated_project( + deal_id + ) - return deal, company, listing + return deal, company, listing, project def get_company_information(self, company_id: str) -> CompanyData: companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api # type: ignore[reportUnknownMemberType] diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index a6e19ef4..1f436eba 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -4,9 +4,11 @@ from datetime import datetime, timezone from typing import Dict, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData +from backend.app.db.models.hubspot_project_data import HubspotProjectData from backend.app.db.models.hubspot_user import HubspotUser from etl.hubspot.company_data import CompanyData from etl.hubspot.hubspotClient import HubspotClient +from etl.hubspot.project_data import ProjectData from etl.hubspot.s3_uploader import S3Uploader from backend.app.db.connection import db_read_session from backend.app.db.models.organisation import Organisation @@ -64,6 +66,30 @@ class HubspotDataToDb: session.commit() return record + def upsert_project(self, project: ProjectData) -> HubspotProjectData: + """Upserts a project record. Updates if project_id exists, otherwise creates new.""" + with db_read_session() as session: + project_id = project["project_id"] + name = project.get("name") + + existing = session.exec( + select(HubspotProjectData).where( + HubspotProjectData.project_id == project_id + ) + ).first() + + if existing: + existing.name = name + existing.updated_at = datetime.now(timezone.utc) + session.add(existing) + record = existing + else: + record = HubspotProjectData(project_id=project_id, name=name) + session.add(record) + + session.commit() + return record + def find_all_deals_with_company_id(self, company_id: str): """Returns a list of deals for a given company_id.""" with db_read_session() as session: @@ -87,6 +113,7 @@ class HubspotDataToDb: company: Optional[str], listing: Optional[dict[str, str]], hubspot_client: HubspotClient, + project: Optional[ProjectData] = None, ): """ Inserts or updates a deal record. @@ -111,7 +138,9 @@ class HubspotDataToDb: self._handle_existing_photo_upload(existing, hubspot_client) print(f"🔄 Updating existing deal (deal_id={deal_id})") - self._update_existing_deal(existing, deal_data, listing, company) + self._update_existing_deal( + existing, deal_data, listing, company, project + ) session.add(existing) session.commit() @@ -121,7 +150,7 @@ class HubspotDataToDb: else: print(f"🆕 Inserting new deal (deal_id={deal_id})") new_record: HubspotDealData = self._build_new_deal( - deal_id, deal_data, listing, company + deal_id, deal_data, listing, company, project ) # Handle upload at insert time @@ -170,10 +199,12 @@ class HubspotDataToDb: deal_data: Dict[str, str], listing: Optional[dict[str, str]], company: Optional[str], + project: Optional[ProjectData] = None, ): for attr, value in { "dealname": deal_data.get("dealname"), "dealstage": deal_data.get("dealstage"), + "project_id": project["project_id"] if project else None, "listing_id": listing.get("listing_id", None) if listing else None, "landlord_property_id": ( listing.get("owner_property_id", None) if listing else None @@ -270,11 +301,13 @@ class HubspotDataToDb: deal_data: Dict[str, str], listing: Optional[dict[str, str]], company: Optional[str], + project: Optional[ProjectData] = None, ) -> HubspotDealData: return HubspotDealData( deal_id=deal_id, dealname=deal_data.get("dealname"), dealstage=deal_data.get("dealstage"), + project_id=project["project_id"] if project else None, listing_id=listing.get("listing_id") if listing else None, landlord_property_id=( listing.get("owner_property_id") if listing else None diff --git a/etl/hubspot/project_data.py b/etl/hubspot/project_data.py new file mode 100644 index 00000000..136298ac --- /dev/null +++ b/etl/hubspot/project_data.py @@ -0,0 +1,6 @@ +from typing import Optional, TypedDict + + +class ProjectData(TypedDict): + project_id: str + name: Optional[str] diff --git a/etl/hubspot/scripts/scraper/local_handler/invoke_local_lambda.py b/etl/hubspot/scripts/scraper/local_handler/invoke_local_lambda.py index 69580a93..03a9ff70 100644 --- a/etl/hubspot/scripts/scraper/local_handler/invoke_local_lambda.py +++ b/etl/hubspot/scripts/scraper/local_handler/invoke_local_lambda.py @@ -14,7 +14,7 @@ payload = { { "task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917", "sub_task_id": "8673913b-1a88-42d7-8578-0449123d94b0", - "hubspot_deal_id": "254427203793", + "hubspot_deal_id": "467396027619", } ) } diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index a7b640cf..176e9b15 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Optional from backend.app.config import get_settings from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb +from etl.hubspot.project_data import ProjectData from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, @@ -34,9 +35,10 @@ def handler(body: dict[str, Any], context: Any) -> None: hubspot_deal: Dict[str, str] company: Optional[str] listing: Optional[dict[str, str]] + project: Optional[ProjectData] - hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing( - hubspot_deal_id + hubspot_deal, company, listing, project = ( + hubspot_client.get_deal_and_company_and_listing_and_project(hubspot_deal_id) ) deal_changed = False @@ -47,7 +49,10 @@ def handler(body: dict[str, Any], context: Any) -> None: db_client: HubspotDataToDb = HubspotDataToDb() db_client.upsert_organisation(company_data) - db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) + if project: + db_client.upsert_project(project) + + db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client, project) # ============================== # Orchestration of other lambdas @@ -77,11 +82,15 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Deal {hubspot_deal_id} has been changed, updating database..." ) + if project: + db_client.upsert_project(project) + db_client.upsert_deal( deal_data=hubspot_deal, company=company, listing=listing, hubspot_client=hubspot_client, + project=project, ) deal_changed = True diff --git a/etl/hubspot/tests/test_hubspot_client_integration.py b/etl/hubspot/tests/test_hubspot_client_integration.py index 0f4b425c..d7fc2d99 100644 --- a/etl/hubspot/tests/test_hubspot_client_integration.py +++ b/etl/hubspot/tests/test_hubspot_client_integration.py @@ -71,7 +71,9 @@ class TestHubspotClientIntegration: def test_get_deal_info_for_db(self, client: HubspotClient): deal_id: str = "263490768079" - deal, company, listing = client.get_deal_and_company_and_listing(deal_id) + deal, company, listing, project = ( + client.get_deal_and_company_and_listing_and_project(deal_id) + ) assert "dealname" in deal assert "dealstage" in deal @@ -81,6 +83,8 @@ class TestHubspotClientIntegration: assert listing is None or "hs_object_id" in listing + assert project is None or "project_id" in project + def test_get_company_information(self, client: HubspotClient): company_id: str = Companies.APPLE.value diff --git a/etl/hubspot/tests/test_hubspot_data_to_db.py b/etl/hubspot/tests/test_hubspot_data_to_db.py index 339e0377..ff4b8294 100644 --- a/etl/hubspot/tests/test_hubspot_data_to_db.py +++ b/etl/hubspot/tests/test_hubspot_data_to_db.py @@ -1,4 +1,5 @@ from etl.hubspot.hubspotDataTodB import HubspotDataToDb +from etl.hubspot.project_data import ProjectData from backend.app.db.models.hubspot_deal_data import HubspotDealData @@ -32,3 +33,55 @@ def test_update_existing_deal__designer_set__overwrites_existing() -> None: ) assert existing.designer == "New Designer" + + +def test_build_new_deal__project_sets_project_id() -> None: + new_deal = _make_instance()._build_new_deal( + deal_id="MOCK_DEAL_ID", + deal_data={}, + listing=None, + company=None, + project=ProjectData(project_id="proj-1", name="Project One"), + ) + + assert new_deal.project_id == "proj-1" + + +def test_build_new_deal__no_project__project_id_none() -> None: + new_deal = _make_instance()._build_new_deal( + deal_id="MOCK_DEAL_ID", + deal_data={}, + listing=None, + company=None, + project=None, + ) + + assert new_deal.project_id is None + + +def test_update_existing_deal__project_sets_project_id() -> None: + existing = HubspotDealData(deal_id="MOCK_DEAL_ID") + + _make_instance()._update_existing_deal( + existing=existing, + deal_data={}, + listing=None, + company=None, + project=ProjectData(project_id="proj-1", name="Project One"), + ) + + assert existing.project_id == "proj-1" + + +def test_update_existing_deal__no_project__clears_project_id() -> None: + existing = HubspotDealData(deal_id="MOCK_DEAL_ID", project_id="old-proj") + + _make_instance()._update_existing_deal( + existing=existing, + deal_data={}, + listing=None, + company=None, + project=None, + ) + + assert existing.project_id is None diff --git a/etl/hubspot/tests/test_scraper_handler.py b/etl/hubspot/tests/test_scraper_handler.py index 4810d171..62d3fe5b 100644 --- a/etl/hubspot/tests/test_scraper_handler.py +++ b/etl/hubspot/tests/test_scraper_handler.py @@ -12,6 +12,7 @@ DEAL_ID = "999" PASHUB_LINK = "https://pashub.example.com/deal/999" MAGICPLAN_QUEUE_URL = "https://sqs.eu-west-2.amazonaws.com/123/magic-plan-dev" PASHUB_QUEUE_URL = "https://sqs.test/pashub" +PROJECT = {"project_id": "proj-1", "name": "Project One"} def make_hubspot_deal(**kwargs: Any) -> Dict[str, Any]: @@ -35,7 +36,8 @@ def run_handler( hubspot_deal: Dict[str, Any], db_deal: Optional[HubspotDealData], listing: Optional[dict], -) -> MagicMock: + project: Optional[dict] = None, +) -> tuple[MagicMock, MagicMock]: mock_sqs = MagicMock() mock_sqs.send_message.return_value = {"MessageId": "test-id"} @@ -47,10 +49,12 @@ def run_handler( ): mock_db_cls.return_value.find_deal_with_deal_id.return_value = db_deal mock_db_cls.return_value.upsert_deal.return_value = None - mock_hs_cls.return_value.get_deal_and_company_and_listing.return_value = ( + mock_hs = mock_hs_cls.return_value + mock_hs.get_deal_and_company_and_listing_and_project.return_value = ( hubspot_deal, None, listing, + project, ) mock_boto3.client.return_value = mock_sqs mock_settings.return_value.MAGICPLAN_SQS_URL = MAGICPLAN_QUEUE_URL @@ -58,7 +62,7 @@ def run_handler( handler.__wrapped__({"hubspot_deal_id": DEAL_ID}, "") - return mock_sqs + return mock_sqs, mock_db_cls.return_value # ==================================== @@ -72,7 +76,7 @@ def test_new_deal_surveyed__sends_magicplan_sqs() -> None: listing = {"national_uprn": UPRN} # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=listing) # Assert mock_sqs.send_message.assert_called_once_with( @@ -88,7 +92,7 @@ def test_new_deal_not_surveyed__no_magicplan_sqs() -> None: hubspot_deal = make_hubspot_deal(outcome="assessed") # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) # Assert mock_sqs.send_message.assert_not_called() @@ -99,7 +103,7 @@ def test_new_deal_surveyed_no_listing__magicplan_sqs_uprn_is_null() -> None: hubspot_deal = make_hubspot_deal(outcome="surveyed") # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) # Assert mock_sqs.send_message.assert_called_once_with( @@ -122,7 +126,7 @@ def test_existing_deal_surveyed_transition__sends_magicplan_sqs() -> None: listing = {"national_uprn": UPRN} # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=listing) # Assert mock_sqs.send_message.assert_called_once_with( @@ -139,7 +143,7 @@ def test_existing_deal_already_surveyed__no_magicplan_sqs() -> None: hubspot_deal = make_hubspot_deal(outcome="surveyed", dealname="New Name") # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) # Assert mock_sqs.send_message.assert_not_called() @@ -155,7 +159,7 @@ def test_new_deal_with_pashub_link__sends_pashub_sqs() -> None: hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) # Assert mock_sqs.send_message.assert_called_once_with( @@ -179,7 +183,7 @@ def test_new_deal_no_pashub_link__no_pashub_sqs() -> None: hubspot_deal = make_hubspot_deal() # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=None, listing=None) # Assert mock_sqs.send_message.assert_not_called() @@ -196,7 +200,7 @@ def test_existing_deal_pashub_link_added__sends_pashub_sqs() -> None: hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK) # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) # Assert mock_sqs.send_message.assert_called_once_with( @@ -221,7 +225,67 @@ def test_existing_deal_pashub_link_unchanged__no_pashub_sqs() -> None: hubspot_deal = make_hubspot_deal(pashub_link=PASHUB_LINK, dealname="New Name") # Act - mock_sqs = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) + mock_sqs, _ = run_handler(hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None) # Assert mock_sqs.send_message.assert_not_called() + + +# ==================================== +# PROJECT upsert +# ==================================== + + +def test_new_deal_with_project__upserts_project() -> None: + # Arrange + hubspot_deal = make_hubspot_deal() + + # Act + _, mock_db = run_handler( + hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=PROJECT + ) + + # Assert + mock_db.upsert_project.assert_called_once_with(PROJECT) + + +def test_new_deal_no_project__no_project_upsert() -> None: + # Arrange + hubspot_deal = make_hubspot_deal() + + # Act + _, mock_db = run_handler( + hubspot_deal=hubspot_deal, db_deal=None, listing=None, project=None + ) + + # Assert + mock_db.upsert_project.assert_not_called() + + +def test_existing_deal_changed_with_project__upserts_project() -> None: + # Arrange + db_deal = make_db_deal(outcome="assessed") + hubspot_deal = make_hubspot_deal(outcome="surveyed") + + # Act + _, mock_db = run_handler( + hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT + ) + + # Assert + mock_db.upsert_project.assert_called_once_with(PROJECT) + + +def test_existing_deal_unchanged__no_project_upsert() -> None: + # Arrange: db deal matches hubspot deal, so the differ reports no change + db_deal = make_db_deal(dealname=DEAL_NAME) + hubspot_deal = make_hubspot_deal() + + # Act + _, mock_db = run_handler( + hubspot_deal=hubspot_deal, db_deal=db_deal, listing=None, project=PROJECT + ) + + # Assert + mock_db.upsert_project.assert_not_called() + mock_db.upsert_deal.assert_not_called() From 17016b299a6a27a99ae8fd38df05cd8f9c91046f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 28 May 2026 12:15:37 +0000 Subject: [PATCH 13/13] booking status --- backend/app/db/models/hubspot_deal_data.py | 1 + etl/hubspot/hubspotClient.py | 1 + etl/hubspot/hubspotDataTodB.py | 2 ++ etl/hubspot/hubspot_deal_differ.py | 1 + 4 files changed, 5 insertions(+) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index b0876c03..2935f2bf 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -24,6 +24,7 @@ class HubspotDealData(SQLModel, table=True): uprn: Optional[str] = Field(default=None) outcome: Optional[str] = Field(default=None) outcome_notes: Optional[str] = Field(default=None) + booking_status: Optional[str] = Field(default=None) major_condition_issue_description: Optional[str] = Field(default=None) major_condition_issue_photos: Optional[str] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 4c542b40..1300b579 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -295,6 +295,7 @@ class HubspotClient: "pipeline", "outcome", "outcome_notes", + "booking_status", "project_code", "major_condition_issue_description", "major_condition_issue_photos", diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 1f436eba..ad5f5c33 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -212,6 +212,7 @@ class HubspotDataToDb: "uprn": listing.get("national_uprn", None) if listing else None, "outcome": deal_data.get("outcome"), "outcome_notes": deal_data.get("outcome_notes"), + "booking_status": deal_data.get("booking_status"), "project_code": deal_data.get("project_code"), "company_id": company, "major_condition_issue_description": deal_data.get( @@ -315,6 +316,7 @@ class HubspotDataToDb: uprn=listing.get("national_uprn") if listing else None, outcome=deal_data.get("outcome"), outcome_notes=deal_data.get("outcome_notes"), + booking_status=deal_data.get("booking_status"), project_code=deal_data.get("project_code"), company_id=company, major_condition_issue_description=deal_data.get( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index dca6e7ea..9df456e6 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -53,6 +53,7 @@ class HubspotDealDiffer: "dealname": "dealname", "project_code": "project_code", "outcome_notes": "outcome_notes", + "booking_status": "booking_status", "major_condition_issue_description": "major_condition_issue_description", "major_condition_issue_photos": "major_condition_issue_photos", "coordination_status__stage_1_": "coordination_status",