From cee4e40bb1cc0f82ec45ba974c2ad63953e856ca Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Wed, 1 Apr 2026 12:10:28 +0000
Subject: [PATCH] latest code

---
 backend/app/db/models/organisation.py    |  2 ++
 etl/hubspot/hubspotClient.py             |  2 ++
 etl/hubspot/hubspotDataTodB.py           | 12 ++++++++++++
 etl/hubspot/scripts/scraper/bulk_load.py |  3 ++-
 etl/hubspot/scripts/scraper/main.py      |  2 +-
 5 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/backend/app/db/models/organisation.py b/backend/app/db/models/organisation.py
index a9718c42..3adc8e9c 100644
--- a/backend/app/db/models/organisation.py
+++ b/backend/app/db/models/organisation.py
@@ -63,6 +63,8 @@ class HubspotDealData(SQLModel, table=True):
     surveyor: Optional[str] = Field(default=None)
     confirmed_survey_date: Optional[datetime] = Field(default=None)
     confirmed_survey_time: Optional[str] = Field(default=None)
+    surveyed_date: Optional[datetime] = Field(default=None)
+    design_type: Optional[str] = Field(default=None)
 
     created_at: datetime = Field(
         sa_column=Column(
diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
index e5461c61..d74a5ed4 100644
--- a/etl/hubspot/hubspotClient.py
+++ b/etl/hubspot/hubspotClient.py
@@ -231,6 +231,8 @@ class HubspotClient:
                 "surveyor",
                 "confirmed_survey_date",
                 "confirmed_survey_time",
+                "surveyed_date",
+                "design_type",
             ],
         )
 
diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py
index 0c38f483..7f06a29d 100644
--- a/etl/hubspot/hubspotDataTodB.py
+++ b/etl/hubspot/hubspotDataTodB.py
@@ -257,6 +257,14 @@ class HubspotDataToDb:
                 deal_in_db.confirmed_survey_time == hs_deal.get("confirmed_survey_time"),
                 "confirmed_survey_time mismatch",
             ),
+            soft_assert(
+                deal_in_db.surveyed_date == self._parse_hs_date(hs_deal.get("surveyed_date")),
+                "surveyed_date mismatch",
+            ),
+            soft_assert(
+                deal_in_db.design_type == hs_deal.get("design_type"),
+                "design_type mismatch",
+            ),
         ]
 
         # If discrepancies found, update from HubSpot
@@ -380,6 +388,8 @@ class HubspotDataToDb:
                     "surveyor": deal_data.get("surveyor"),
                     "confirmed_survey_date": self._parse_hs_date(deal_data.get("confirmed_survey_date")),
                     "confirmed_survey_time": deal_data.get("confirmed_survey_time"),
+                    "surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")),
+                    "design_type": deal_data.get("design_type"),
                 }.items():
                     setattr(existing, attr, value or getattr(existing, attr))
 
@@ -462,6 +472,8 @@ class HubspotDataToDb:
                     surveyor=deal_data.get("surveyor"),
                     confirmed_survey_date=self._parse_hs_date(deal_data.get("confirmed_survey_date")),
                     confirmed_survey_time=deal_data.get("confirmed_survey_time"),
+                    surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
+                    design_type=deal_data.get("design_type"),
                 )
 
                 # Handle upload at insert time
diff --git a/etl/hubspot/scripts/scraper/bulk_load.py b/etl/hubspot/scripts/scraper/bulk_load.py
index 6fac23ea..5dc9570e 100644
--- a/etl/hubspot/scripts/scraper/bulk_load.py
+++ b/etl/hubspot/scripts/scraper/bulk_load.py
@@ -1,6 +1,7 @@
 from etl.hubspot.hubspotClient import HubspotClient, Companies, Pipeline
 from etl.hubspot.scripts.scraper.main import handler
 from tqdm import tqdm
+import json
 
 
 PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
@@ -29,7 +30,7 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
                     continue
 
                 deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
-                handler({"hubspot_deal_id": deal_id}, context=None)
+                handler({"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]}, context=None)
                 processed += 1
                 deal_bar.set_postfix({"status": "done", "deal": deal_id})
 
diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py
index 459ea5c2..4525c8cb 100644
--- a/etl/hubspot/scripts/scraper/main.py
+++ b/etl/hubspot/scripts/scraper/main.py
@@ -21,7 +21,7 @@ def handler(event: dict[str, Any], context: Any, local: bool = False) -> None:
                 {
                     "body": json.dumps(
                         {
-                            "hubspot_deal_id": "409487859944",
+                            "hubspot_deal_id": "483651713260",
                         }
                     )
                 }