Merge pull request #946 from Hestia-Homes/feature/hubspot_etl

add design type and surveyed date per request

backend/app/db/models/organisation.py

@@ -63,6 +63,8 @@ class HubspotDealData(SQLModel, table=True):
     surveyor: Optional[str] = Field(default=None)
     confirmed_survey_date: Optional[datetime] = Field(default=None)
     confirmed_survey_time: Optional[str] = Field(default=None)
+    surveyed_date: Optional[datetime] = Field(default=None)
+    design_type: Optional[str] = Field(default=None)
 
     created_at: datetime = Field(
         sa_column=Column(

etl/hubspot/hubspotClient.py

@@ -231,6 +231,8 @@ class HubspotClient:
                 "surveyor",
                 "confirmed_survey_date",
                 "confirmed_survey_time",
+                "surveyed_date",
+                "design_type",
             ],
         )
 

etl/hubspot/hubspotDataTodB.py

@@ -257,6 +257,14 @@ class HubspotDataToDb:
                 deal_in_db.confirmed_survey_time == hs_deal.get("confirmed_survey_time"),
                 "confirmed_survey_time mismatch",
             ),
+            soft_assert(
+                deal_in_db.surveyed_date == self._parse_hs_date(hs_deal.get("surveyed_date")),
+                "surveyed_date mismatch",
+            ),
+            soft_assert(
+                deal_in_db.design_type == hs_deal.get("design_type"),
+                "design_type mismatch",
+            ),
         ]
 
         # If discrepancies found, update from HubSpot
@@ -380,6 +388,8 @@ class HubspotDataToDb:
                     "surveyor": deal_data.get("surveyor"),
                     "confirmed_survey_date": self._parse_hs_date(deal_data.get("confirmed_survey_date")),
                     "confirmed_survey_time": deal_data.get("confirmed_survey_time"),
+                    "surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")),
+                    "design_type": deal_data.get("design_type"),
                 }.items():
                     setattr(existing, attr, value or getattr(existing, attr))
 
@@ -462,6 +472,8 @@ class HubspotDataToDb:
                     surveyor=deal_data.get("surveyor"),
                     confirmed_survey_date=self._parse_hs_date(deal_data.get("confirmed_survey_date")),
                     confirmed_survey_time=deal_data.get("confirmed_survey_time"),
+                    surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
+                    design_type=deal_data.get("design_type"),
                 )
 
                 # Handle upload at insert time

etl/hubspot/scripts/scraper/bulk_load.py

@@ -1,6 +1,7 @@
 from etl.hubspot.hubspotClient import HubspotClient, Companies, Pipeline
 from etl.hubspot.scripts.scraper.main import handler
 from tqdm import tqdm
+import json
 
 
 PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
@@ -29,7 +30,7 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
                     continue
 
                 deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
-                handler({"hubspot_deal_id": deal_id}, context=None)
+                handler({"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]}, context=None)
                 processed += 1
                 deal_bar.set_postfix({"status": "done", "deal": deal_id})
 

etl/hubspot/scripts/scraper/main.py

@@ -21,7 +21,7 @@ def handler(event: dict[str, Any], context: Any, local: bool = False) -> None:
                 {
                     "body": json.dumps(
                         {
-                            "hubspot_deal_id": "409487859944",
+                            "hubspot_deal_id": "483651713260",
                         }
                     )
                 }