From 42ea70e60fcf57053d492f8d63fbfae2f761328d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 20 Apr 2026 15:23:38 +0000 Subject: [PATCH 1/2] added additional deal properties --- backend/app/db/models/hubspot_deal_data.py | 4 ++++ etl/hubspot/hubspotClient.py | 8 ++++++-- etl/hubspot/hubspotDataTodB.py | 20 ++++++++++++++++---- etl/hubspot/hubspot_deal_differ.py | 8 ++++++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 758f688d..27649042 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -38,6 +38,10 @@ class HubspotDealData(SQLModel, table=True): dampmould_growth: Optional[str] = Field(default=None) damp_mould_and_repairs_comments: Optional[str] = Field(default=None) pre_sap: Optional[str] = Field(default=None) + batch: Optional[str] = Field(default=None) + block_reference: Optional[str] = Field(default=None) + epc_prn: Optional[str] = Field(default=None) + potential_post_sap_score_dropdown: Optional[str] = Field(default=None) coordinator: Optional[str] = Field(default=None) mtp_completion_date: Optional[datetime] = Field(default=None) mtp_re_model_completion_date: Optional[datetime] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 6bdf71ed..3a562700 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -254,12 +254,12 @@ class HubspotClient: "sharepoint_link", "dampmould_growth", "damp_mould_and_repairs_comments", - "pre_sap", + "pre_sap_score_dropdown", "coordinator", "mtp_completion_date", "mtp_re_model_completion_date", "ioe_v3_completion_date", - "proposed_measures", + "proposed_measures_dropdown", "approved_package", "designer", "design_completion_date", @@ -275,6 +275,10 @@ class HubspotClient: "confirmed_survey_time", "surveyed_date", "design_type", + "batch", + "block_reference", + "epc_prn", + "potential_post_sap_score_dropdown", ], ) ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 9756833b..eb8e2d14 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -159,7 +159,13 @@ class HubspotDataToDb: "damp_mould_and_repairs_comments": deal_data.get( "damp_mould_and_repairs_comments" ), - "pre_sap": deal_data.get("pre_sap"), + "pre_sap": deal_data.get("pre_sap_score_dropdown"), + "batch": deal_data.get("batch"), + "block_reference": deal_data.get("block_reference"), + "epc_prn": deal_data.get("epc_prn"), + "potential_post_sap_score_dropdown": deal_data.get( + "potential_post_sap_score_dropdown" + ), "coordinator": deal_data.get("coordinator"), "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), "mtp_re_model_completion_date": parse_hs_date( @@ -168,7 +174,7 @@ class HubspotDataToDb: "ioe_v3_completion_date": parse_hs_date( deal_data.get("ioe_v3_completion_date") ), - "proposed_measures": deal_data.get("proposed_measures"), + "proposed_measures": deal_data.get("proposed_measures_dropdown"), "approved_package": deal_data.get("approved_package"), "designer": deal_data.get("designer"), "design_completion_date": parse_hs_date( @@ -228,7 +234,13 @@ class HubspotDataToDb: damp_mould_and_repairs_comments=deal_data.get( "damp_mould_and_repairs_comments" ), - pre_sap=deal_data.get("pre_sap"), + pre_sap=deal_data.get("pre_sap_score_dropdown"), + batch=deal_data.get("batch"), + block_reference=deal_data.get("block_reference"), + epc_prn=deal_data.get("epc_prn"), + potential_post_sap_score_dropdown=deal_data.get( + "potential_post_sap_score_dropdown" + ), coordinator=deal_data.get("coordinator"), mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), mtp_re_model_completion_date=parse_hs_date( @@ -237,7 +249,7 @@ class HubspotDataToDb: ioe_v3_completion_date=parse_hs_date( deal_data.get("ioe_v3_completion_date") ), - proposed_measures=deal_data.get("proposed_measures"), + proposed_measures=deal_data.get("proposed_measures_dropdown"), approved_package=deal_data.get("approved_package"), designer=deal_data.get("designer"), design_completion_date=parse_hs_date( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 74c8264d..cf9ad1ee 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -62,9 +62,13 @@ class HubspotDealDiffer: "sharepoint_link": "sharepoint_link", "dampmould_growth": "dampmould_growth", "damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments", - "pre_sap": "pre_sap", + "pre_sap_score_dropdown": "pre_sap", + "batch": "batch", + "block_reference": "block_reference", + "epc_prn": "epc_prn", + "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", "coordinator": "coordinator", - "proposed_measures": "proposed_measures", + "proposed_measures_dropdown": "proposed_measures", "approved_package": "approved_package", "designer": "designer", "actual_measures_installed": "actual_measures_installed", From 902e5cd8dfb0696b80190eb66d7a96f44f8f46fc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 20 Apr 2026 15:44:54 +0000 Subject: [PATCH 2/2] added more deal properties --- backend/app/db/models/hubspot_deal_data.py | 4 ++++ etl/hubspot/hubspotClient.py | 4 ++++ etl/hubspot/hubspotDataTodB.py | 8 ++++++++ etl/hubspot/hubspot_deal_differ.py | 4 ++++ etl/hubspot/scripts/scraper/bulk_load.py | 4 ++-- 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index 27649042..fa508fbe 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -42,6 +42,10 @@ class HubspotDealData(SQLModel, table=True): block_reference: Optional[str] = Field(default=None) epc_prn: Optional[str] = Field(default=None) potential_post_sap_score_dropdown: Optional[str] = Field(default=None) + ei_score: Optional[str] = Field(default=None) + ei_score__potential_: Optional[str] = Field(default=None) + epc_sap_score: Optional[str] = Field(default=None) + epc_sap_score__potential_: Optional[str] = Field(default=None) coordinator: Optional[str] = Field(default=None) mtp_completion_date: Optional[datetime] = Field(default=None) mtp_re_model_completion_date: Optional[datetime] = Field(default=None) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index 3a562700..df28e4d6 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -279,6 +279,10 @@ class HubspotClient: "block_reference", "epc_prn", "potential_post_sap_score_dropdown", + "ei_score", + "ei_score__potential_", + "epc_sap_score", + "epc_sap_score__potential_", ], ) ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index eb8e2d14..a2eb24c2 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -166,6 +166,10 @@ class HubspotDataToDb: "potential_post_sap_score_dropdown": deal_data.get( "potential_post_sap_score_dropdown" ), + "ei_score": deal_data.get("ei_score"), + "ei_score__potential_": deal_data.get("ei_score__potential_"), + "epc_sap_score": deal_data.get("epc_sap_score"), + "epc_sap_score__potential_": deal_data.get("epc_sap_score__potential_"), "coordinator": deal_data.get("coordinator"), "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), "mtp_re_model_completion_date": parse_hs_date( @@ -241,6 +245,10 @@ class HubspotDataToDb: potential_post_sap_score_dropdown=deal_data.get( "potential_post_sap_score_dropdown" ), + ei_score=deal_data.get("ei_score"), + ei_score__potential_=deal_data.get("ei_score__potential_"), + epc_sap_score=deal_data.get("epc_sap_score"), + epc_sap_score__potential_=deal_data.get("epc_sap_score__potential_"), coordinator=deal_data.get("coordinator"), mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), mtp_re_model_completion_date=parse_hs_date( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index cf9ad1ee..80c1fe04 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -67,6 +67,10 @@ class HubspotDealDiffer: "block_reference": "block_reference", "epc_prn": "epc_prn", "potential_post_sap_score_dropdown": "potential_post_sap_score_dropdown", + "ei_score": "ei_score", + "ei_score__potential_": "ei_score__potential_", + "epc_sap_score": "epc_sap_score", + "epc_sap_score__potential_": "epc_sap_score__potential_", "coordinator": "coordinator", "proposed_measures_dropdown": "proposed_measures", "approved_package": "approved_package", diff --git a/etl/hubspot/scripts/scraper/bulk_load.py b/etl/hubspot/scripts/scraper/bulk_load.py index f0529905..91aa89e2 100644 --- a/etl/hubspot/scripts/scraper/bulk_load.py +++ b/etl/hubspot/scripts/scraper/bulk_load.py @@ -9,8 +9,8 @@ PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value companies = list( [ # Companies.THE_GUINESS_PARTNERSHIP, - Companies.SOUTHERN_HOUSING_GROUP, - # Companies.CALICO_HOMES, + # Companies.SOUTHERN_HOUSING_GROUP, + Companies.CALICO_HOMES, ] )