modified hubspot etl to fetch new records and replace coordination and designer with their assigned variants

etl/hubspot/hubspotClient.py

@@ -255,13 +255,13 @@ class HubspotClient:
                     "dampmould_growth",
                     "damp_mould_and_repairs_comments",
                     "pre_sap_score_dropdown",
-                    "coordinator",
+                    "assigned_coordinator",
                     "mtp_completion_date",
                     "mtp_re_model_completion_date",
                     "ioe_v3_completion_date",
                     "proposed_measures_dropdown",
                     "approved_package",
-                    "designer",
+                    "assigned_designer",
                     "design_completion_date",
                     "actual_measures_installed",
                     "installer",
@@ -283,6 +283,16 @@ class HubspotClient:
                     "ei_score__potential_",
                     "epc_sap_score",
                     "epc_sap_score__potential_",
+                    "survey_type",
+                    "measures_for_pibi_ordered",
+                    "pibi_order_date",
+                    "pibi_completed_date",
+                    "property_halted_date",
+                    "property_halted_reason",
+                    "technical_approved_measures_for_install",
+                    "sent_to_iw_for_pricing",
+                    "osmosis_survey_required",
+                    "osmosis_survey_date",
                 ],
             )
         )

etl/hubspot/hubspotDataTodB.py

@@ -9,7 +9,7 @@ from etl.hubspot.hubspotClient import HubspotClient
 from etl.hubspot.s3_uploader import S3Uploader
 from backend.app.db.connection import db_read_session
 from backend.app.db.models.organisation import Organisation
-from etl.hubspot.utils import parse_hs_date
+from etl.hubspot.utils import parse_hs_bool, parse_hs_date
 from utils.logger import setup_logger
 
 
@@ -170,7 +170,7 @@ class HubspotDataToDb:
             "ei_score__potential_": deal_data.get("ei_score__potential_"),
             "epc_sap_score": deal_data.get("epc_sap_score"),
             "epc_sap_score__potential_": deal_data.get("epc_sap_score__potential_"),
-            "coordinator": deal_data.get("coordinator"),
+            "coordinator": deal_data.get("assigned_coordinator"),
             "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")),
             "mtp_re_model_completion_date": parse_hs_date(
                 deal_data.get("mtp_re_model_completion_date")
@@ -180,7 +180,7 @@ class HubspotDataToDb:
             ),
             "proposed_measures": deal_data.get("proposed_measures_dropdown"),
             "approved_package": deal_data.get("approved_package"),
-            "designer": deal_data.get("designer"),
+            "designer": deal_data.get("assigned_designer"),
             "design_completion_date": parse_hs_date(
                 deal_data.get("design_completion_date")
             ),
@@ -202,6 +202,24 @@ class HubspotDataToDb:
             "confirmed_survey_time": deal_data.get("confirmed_survey_time"),
             "surveyed_date": parse_hs_date(deal_data.get("surveyed_date")),
             "design_type": deal_data.get("design_type"),
+            "survey_type": deal_data.get("survey_type"),
+            "measures_for_pibi_ordered": deal_data.get("measures_for_pibi_ordered"),
+            "pibi_order_date": parse_hs_date(deal_data.get("pibi_order_date")),
+            "pibi_completed_date": parse_hs_date(deal_data.get("pibi_completed_date")),
+            "property_halted_date": parse_hs_date(
+                deal_data.get("property_halted_date")
+            ),
+            "property_halted_reason": deal_data.get("property_halted_reason"),
+            "technical_approved_measures_for_install": deal_data.get(
+                "technical_approved_measures_for_install"
+            ),
+            "sent_to_installer_for_pricing": parse_hs_date(
+                deal_data.get("sent_to_iw_for_pricing")
+            ),
+            "domna_survey_required": parse_hs_bool(
+                deal_data.get("osmosis_survey_required")
+            ),
+            "domna_survey_date": parse_hs_date(deal_data.get("osmosis_survey_date")),
         }.items():
             setattr(existing, attr, value or getattr(existing, attr))
 
@@ -249,7 +267,7 @@ class HubspotDataToDb:
             ei_score__potential_=deal_data.get("ei_score__potential_"),
             epc_sap_score=deal_data.get("epc_sap_score"),
             epc_sap_score__potential_=deal_data.get("epc_sap_score__potential_"),
-            coordinator=deal_data.get("coordinator"),
+            coordinator=deal_data.get("assigned_coordinator"),
             mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")),
             mtp_re_model_completion_date=parse_hs_date(
                 deal_data.get("mtp_re_model_completion_date")
@@ -259,7 +277,7 @@ class HubspotDataToDb:
             ),
             proposed_measures=deal_data.get("proposed_measures_dropdown"),
             approved_package=deal_data.get("approved_package"),
-            designer=deal_data.get("designer"),
+            designer=deal_data.get("assigned_designer"),
             design_completion_date=parse_hs_date(
                 deal_data.get("design_completion_date")
             ),
@@ -279,6 +297,22 @@ class HubspotDataToDb:
             confirmed_survey_time=deal_data.get("confirmed_survey_time"),
             surveyed_date=parse_hs_date(deal_data.get("surveyed_date")),
             design_type=deal_data.get("design_type"),
+            survey_type=deal_data.get("survey_type"),
+            measures_for_pibi_ordered=deal_data.get("measures_for_pibi_ordered"),
+            pibi_order_date=parse_hs_date(deal_data.get("pibi_order_date")),
+            pibi_completed_date=parse_hs_date(deal_data.get("pibi_completed_date")),
+            property_halted_date=parse_hs_date(deal_data.get("property_halted_date")),
+            property_halted_reason=deal_data.get("property_halted_reason"),
+            technical_approved_measures_for_install=deal_data.get(
+                "technical_approved_measures_for_install"
+            ),
+            sent_to_installer_for_pricing=parse_hs_date(
+                deal_data.get("sent_to_iw_for_pricing")
+            ),
+            domna_survey_required=parse_hs_bool(
+                deal_data.get("osmosis_survey_required")
+            ),
+            domna_survey_date=parse_hs_date(deal_data.get("osmosis_survey_date")),
         )
 
     def _handle_existing_photo_upload(

etl/hubspot/hubspot_deal_differ.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Optional
 
 from backend.app.db.models.hubspot_deal_data import HubspotDealData
-from etl.hubspot.utils import parse_hs_date
+from etl.hubspot.utils import parse_hs_bool, parse_hs_date
 
 
 class HubspotDealDiffer:
@@ -71,10 +71,10 @@ class HubspotDealDiffer:
             "ei_score__potential_": "ei_score__potential_",
             "epc_sap_score": "epc_sap_score",
             "epc_sap_score__potential_": "epc_sap_score__potential_",
-            "coordinator": "coordinator",
+            "assigned_coordinator": "coordinator",
             "proposed_measures_dropdown": "proposed_measures",
             "approved_package": "approved_package",
-            "designer": "designer",
+            "assigned_designer": "designer",
             "actual_measures_installed": "actual_measures_installed",
             "installer": "installer",
             "installer_handover": "installer_handover",
@@ -82,6 +82,10 @@ class HubspotDealDiffer:
             "design_type": "design_type",
             "surveyor": "surveyor",
             "confirmed_survey_time": "confirmed_survey_time",
+            "survey_type": "survey_type",
+            "measures_for_pibi_ordered": "measures_for_pibi_ordered",
+            "property_halted_reason": "property_halted_reason",
+            "technical_approved_measures_for_install": "technical_approved_measures_for_install",
         }
 
         for hs_field, db_field in FIELD_MAP.items():
@@ -102,6 +106,11 @@ class HubspotDealDiffer:
             ("expected_commencement_date", "expected_commencement_date"),
             ("confirmed_survey_date", "confirmed_survey_date"),
             ("surveyed_date", "surveyed_date"),
+            ("pibi_order_date", "pibi_order_date"),
+            ("pibi_completed_date", "pibi_completed_date"),
+            ("property_halted_date", "property_halted_date"),
+            ("sent_to_iw_for_pricing", "sent_to_installer_for_pricing"),
+            ("osmosis_survey_date", "domna_survey_date"),
         ]
 
         for hs_field, db_field in date_fields:
@@ -111,6 +120,18 @@ class HubspotDealDiffer:
             if old_value != new_value:
                 return True
 
+        # --- Boolean fields ---
+        bool_fields = [
+            ("osmosis_survey_required", "domna_survey_required"),
+        ]
+
+        for hs_field, db_field in bool_fields:
+            old_value = getattr(old_deal, db_field)
+            new_value = parse_hs_bool(new_deal.get(hs_field))
+
+            if old_value != new_value:
+                return True
+
         # --- Time field ---
         if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"):
             return True

etl/hubspot/utils.py

@@ -14,3 +14,11 @@ def parse_hs_date(value: Optional[str]) -> Optional[datetime]:
         return dt.astimezone(timezone.utc)
     except ValueError:
         return None
+
+
+def parse_hs_bool(value: Optional[str]) -> Optional[bool]:
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() == "true"