junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #946 from Hestia-Homes/feature/hubspot_etl

Browse source 
add design type and surveyed date per request
This commit is contained in:
Jun-te Kim 2026-04-01 13:50:06 +01:00 • committed by GitHub
commit 91ae897946
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 19 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/app/db/models/organisation.py
View file

@ -63,6 +63,8 @@ class HubspotDealData(SQLModel, table=True):
surveyor: Optional[str] = Field(default=None)
confirmed_survey_date: Optional[datetime] = Field(default=None)
confirmed_survey_time: Optional[str] = Field(default=None)
surveyed_date: Optional[datetime] = Field(default=None)
design_type: Optional[str] = Field(default=None)
created_at: datetime = Field(
sa_column=Column(

2
etl/hubspot/hubspotClient.py
View file

@ -231,6 +231,8 @@ class HubspotClient:
"surveyor",
"confirmed_survey_date",
"confirmed_survey_time",
"surveyed_date",
"design_type",
],
)

12
etl/hubspot/hubspotDataTodB.py
View file

@ -257,6 +257,14 @@ class HubspotDataToDb:
deal_in_db.confirmed_survey_time == hs_deal.get("confirmed_survey_time"),
"confirmed_survey_time mismatch",
),
soft_assert(
deal_in_db.surveyed_date == self._parse_hs_date(hs_deal.get("surveyed_date")),
"surveyed_date mismatch",
),
soft_assert(
deal_in_db.design_type == hs_deal.get("design_type"),
"design_type mismatch",
),
]
# If discrepancies found, update from HubSpot
@ -380,6 +388,8 @@ class HubspotDataToDb:
"surveyor": deal_data.get("surveyor"),
"confirmed_survey_date": self._parse_hs_date(deal_data.get("confirmed_survey_date")),
"confirmed_survey_time": deal_data.get("confirmed_survey_time"),
"surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")),
"design_type": deal_data.get("design_type"),
}.items():
setattr(existing, attr, value or getattr(existing, attr))
@ -462,6 +472,8 @@ class HubspotDataToDb:
surveyor=deal_data.get("surveyor"),
confirmed_survey_date=self._parse_hs_date(deal_data.get("confirmed_survey_date")),
confirmed_survey_time=deal_data.get("confirmed_survey_time"),
surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
design_type=deal_data.get("design_type"),
)
# Handle upload at insert time

3
etl/hubspot/scripts/scraper/bulk_load.py
View file

@ -1,6 +1,7 @@
from etl.hubspot.hubspotClient import HubspotClient, Companies, Pipeline
from etl.hubspot.scripts.scraper.main import handler
from tqdm import tqdm
import json
PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
@ -29,7 +30,7 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
continue
deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
handler({"hubspot_deal_id": deal_id}, context=None)
handler({"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]}, context=None)
processed += 1
deal_bar.set_postfix({"status": "done", "deal": deal_id})

2
etl/hubspot/scripts/scraper/main.py
View file

@ -21,7 +21,7 @@ def handler(event: dict[str, Any], context: Any, local: bool = False) -> None:
{
"body": json.dumps(
{
"hubspot_deal_id": "409487859944",
"hubspot_deal_id": "483651713260",
}
)
}