From 57a531fc04c7e8f357783cc1964d9107e90d8712 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 30 Jul 2026 10:01:08 +0000 Subject: [PATCH] Sync the epc_lodged_deal Yes/No dropdown into the HubSpot deal ETL Adds parse_hs_yes_no rather than reusing parse_hs_bool: that one reads a HubSpot checkbox ("true"/"false") and would call "Yes" false. Anything that is neither yes nor no parses to None, so an unanswered dropdown never reads as "answered No". The differ gets its own yes_no_fields block alongside bool_fields, since the two lists need different parsers. The epc_lodged_deal column still has to be created by a migration in Hestia-Homes/assessment-model before this can write. Co-Authored-By: Claude Opus 5 --- backend/app/db/models/hubspot_deal_data.py | 3 + etl/hubspot/hubspotClient.py | 1 + etl/hubspot/hubspotDataTodB.py | 4 +- etl/hubspot/hubspot_deal_differ.py | 19 +++++- etl/hubspot/tests/test_hubspot_data_to_db.py | 62 +++++++++++++++++++ etl/hubspot/tests/test_hubspot_deal_differ.py | 32 ++++++++++ etl/hubspot/utils.py | 18 ++++++ 7 files changed, 137 insertions(+), 2 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 966966a4c..d82f0ec12 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -50,6 +50,9 @@ class HubspotDealData(SQLModel, table=True): block_reference: Optional[str] = Field(default=None) nonfunded_measures: Optional[str] = Field(default=None) epc_prn: Optional[str] = Field(default=None) + # HubSpot carries this as a "Yes"/"No" dropdown (property id + # epc_lodged_deal); stored as a bool, with None for "not answered". + epc_lodged_deal: Optional[bool] = Field(default=None) potential_post_sap_score_dropdown: Optional[str] = Field(default=None) ei_score: Optional[str] = Field(default=None) ei_score__potential_: Optional[str] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 76964bdd4..e082d9c77 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -303,6 +303,7 @@ class HubspotClient: "block_reference", "nonfunded_measures", "epc_prn", + "epc_lodged_deal", "potential_post_sap_score_dropdown", "ei_score", "ei_score__potential_", diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index cb3a2297e..57394be77 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -12,7 +12,7 @@ from etl.hubspot.project_data import ProjectData from etl.hubspot.s3_uploader import S3Uploader from backend.app.db.connection import db_read_session from backend.app.db.models.organisation import Organisation -from etl.hubspot.utils import parse_hs_bool, parse_hs_date +from etl.hubspot.utils import parse_hs_bool, parse_hs_date, parse_hs_yes_no from utils.logger import setup_logger logger = setup_logger() @@ -238,6 +238,7 @@ class HubspotDataToDb: "block_reference": deal_data.get("block_reference"), "nonfunded_measures": deal_data.get("nonfunded_measures"), "epc_prn": deal_data.get("epc_prn"), + "epc_lodged_deal": parse_hs_yes_no(deal_data.get("epc_lodged_deal")), "potential_post_sap_score_dropdown": deal_data.get( "potential_post_sap_score_dropdown" ), @@ -363,6 +364,7 @@ class HubspotDataToDb: block_reference=deal_data.get("block_reference"), nonfunded_measures=deal_data.get("nonfunded_measures"), epc_prn=deal_data.get("epc_prn"), + epc_lodged_deal=parse_hs_yes_no(deal_data.get("epc_lodged_deal")), potential_post_sap_score_dropdown=deal_data.get( "potential_post_sap_score_dropdown" ), diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index ec07af5e6..9dac25064 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -4,7 +4,12 @@ from typing import Any, Dict, List, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData from domain.abri.models import UNSUCCESSFUL_OUTCOMES from etl.hubspot.project_data import ProjectData -from etl.hubspot.utils import parse_hs_bool, parse_hs_date, parse_hs_int +from etl.hubspot.utils import ( + parse_hs_bool, + parse_hs_date, + parse_hs_int, + parse_hs_yes_no, +) from utilities.logger import setup_logger logger = setup_logger() @@ -167,6 +172,18 @@ class HubspotDealDiffer: if old_value != new_value: return True + # --- Yes/No dropdown fields --- + yes_no_fields = [ + ("epc_lodged_deal", "epc_lodged_deal"), + ] + + for hs_field, db_field in yes_no_fields: + old_value = getattr(old_deal, db_field) + new_value = parse_hs_yes_no(new_deal.get(hs_field)) + + if old_value != new_value: + return True + # --- Time field --- if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"): return True diff --git a/etl/hubspot/tests/test_hubspot_data_to_db.py b/etl/hubspot/tests/test_hubspot_data_to_db.py index 9f1f94ccd..19a353704 100644 --- a/etl/hubspot/tests/test_hubspot_data_to_db.py +++ b/etl/hubspot/tests/test_hubspot_data_to_db.py @@ -195,3 +195,65 @@ def test_update_existing_deal__planning_fields_overwritten() -> None: assert existing.planning_comments == "New Comments" assert existing.planning_status == "New Status" assert existing.planning_suggested_approach == "New Approach" + + +def test_build_new_deal__epc_lodged_yes__is_true() -> None: + new_deal = _make_instance()._build_new_deal( + deal_id="MOCK_DEAL_ID", + deal_data={"epc_lodged_deal": "Yes"}, + listing=None, + company=None, + project=None, + ) + + assert new_deal.epc_lodged_deal is True + + +def test_build_new_deal__epc_lodged_no__is_false() -> None: + new_deal = _make_instance()._build_new_deal( + deal_id="MOCK_DEAL_ID", + deal_data={"epc_lodged_deal": "No"}, + listing=None, + company=None, + project=None, + ) + + assert new_deal.epc_lodged_deal is False + + +def test_build_new_deal__epc_lodged_unset__is_none() -> None: + new_deal = _make_instance()._build_new_deal( + deal_id="MOCK_DEAL_ID", + deal_data={}, + listing=None, + company=None, + project=None, + ) + + assert new_deal.epc_lodged_deal is None + + +def test_update_existing_deal__epc_lodged_no__overwrites_true() -> None: + existing = HubspotDealData(deal_id="MOCK_DEAL_ID", epc_lodged_deal=True) + + _make_instance()._update_existing_deal( + existing=existing, + deal_data={"epc_lodged_deal": "No"}, + listing=None, + company=None, + ) + + assert existing.epc_lodged_deal is False + + +def test_update_existing_deal__epc_lodged_cleared__nulls_the_flag() -> None: + existing = HubspotDealData(deal_id="MOCK_DEAL_ID", epc_lodged_deal=True) + + _make_instance()._update_existing_deal( + existing=existing, + deal_data={"epc_lodged_deal": ""}, + listing=None, + company=None, + ) + + assert existing.epc_lodged_deal is None diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index e851b206b..5776d1e93 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -1181,6 +1181,38 @@ def test_db_update_trigger__no_changes__returns_false() -> None: assert result is False +def test_db_update_trigger__epc_lodged_unchanged__returns_false() -> None: + deal_id = uuid.uuid4() + + old_deal = make_old_deal(id=deal_id, epc_lodged_deal=True) + new_deal = make_new_deal(deal_id, hs_object_id="1", epc_lodged_deal="Yes") + + result = HubspotDealDiffer.check_for_db_update_trigger( + new_deal=new_deal, + new_company=None, + new_listing=None, + old_deal=old_deal, + ) + + assert result is False + + +def test_db_update_trigger__epc_lodged_flipped_to_no__returns_true() -> None: + deal_id = uuid.uuid4() + + old_deal = make_old_deal(id=deal_id, epc_lodged_deal=True) + new_deal = make_new_deal(deal_id, hs_object_id="1", epc_lodged_deal="No") + + result = HubspotDealDiffer.check_for_db_update_trigger( + new_deal=new_deal, + new_company=None, + new_listing=None, + old_deal=old_deal, + ) + + assert result is True + + def test_db_update_trigger__dealname_changed__returns_true() -> None: deal_id = uuid.uuid4() diff --git a/etl/hubspot/utils.py b/etl/hubspot/utils.py index dea173324..2ee017543 100644 --- a/etl/hubspot/utils.py +++ b/etl/hubspot/utils.py @@ -31,3 +31,21 @@ def parse_hs_bool(value: Optional[str]) -> Optional[bool]: if isinstance(value, bool): return value return str(value).strip().lower() == "true" + + +def parse_hs_yes_no(value: Optional[str]) -> Optional[bool]: + """Reads a HubSpot 'Yes'/'No' dropdown as a bool. + + Distinct from parse_hs_bool, which reads a HubSpot checkbox + ("true"/"false") and would call "Yes" false. Anything that is neither + yes nor no — unset, blank, or an option we don't know — is None rather + than False, so "not answered" never reads as "answered No". + """ + if value is None: + return None + normalised = str(value).strip().lower() + if normalised == "yes": + return True + if normalised == "no": + return False + return None