junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

diff checker for db load trigger 🟩

Browse source
This commit is contained in:
Daniel Roth 2026-04-09 11:19:33 +00:00
parent 125527baa9
commit 36aaabb3cf
5 changed files with 127 additions and 249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
backend/app/db/models/hubspot_deal_data.py
View file

@ -59,19 +59,21 @@ class HubspotDealData(SQLModel, table=True):
surveyed_date: Optional[datetime] = Field(default=None) surveyed_date: Optional[datetime] = Field(default=None)
design_type: Optional[str] = Field(default=None) design_type: Optional[str] = Field(default=None)
created_at: datetime = Field( created_at: Optional[datetime] = Field(
sa_column=Column( sa_column=Column(
DateTime(timezone=True), DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"), server_default=text("(NOW() AT TIME ZONE 'utc')"),
nullable=False, nullable=False,
) ),
default=None, # Nullable in db but optional here as value is set on db save for new record
) )
updated_at: datetime = Field( updated_at: Optional[datetime] = Field(
sa_column=Column( sa_column=Column(
DateTime(timezone=True), DateTime(timezone=True),
server_default=text("(NOW() AT TIME ZONE 'utc')"), server_default=text("(NOW() AT TIME ZONE 'utc')"),
onupdate=func.now(), onupdate=func.now(),
nullable=False, nullable=False,
) ),
default=None, # Nullable in db but optional here as value is set on db save for new record
) )

247
etl/hubspot/hubspotDataTodB.py
View file

@ -10,6 +10,7 @@ from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.s3_uploader import S3Uploader from etl.hubspot.s3_uploader import S3Uploader
from backend.app.db.connection import db_read_session from backend.app.db.connection import db_read_session
from backend.app.db.models.organisation import Organisation from backend.app.db.models.organisation import Organisation
from etl.hubspot.utils import parse_hs_date
class HubspotDataToDb: class HubspotDataToDb:
@ -60,11 +61,7 @@ class HubspotDataToDb:
session.commit() session.commit()
return record return record
def new_record_to_hubspot_data(self, deal_data, company, listing, hubspot_client): def find_all_deals_with_company_id(self, company_id: str):
print("⚠️ Deprecated — use the new interface instead.")
return self.upsert_deal(deal_data, company, listing, hubspot_client)
def find_all_deals_with_company_id(self, company_id):
"""Returns a list of deals for a given company_id.""" """Returns a list of deals for a given company_id."""
with db_read_session() as session: with db_read_session() as session:
return ( return (
@ -137,7 +134,7 @@ class HubspotDataToDb:
return False return False
else: else:
print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}") print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}")
else: else:
print(f"✅ No update or upload required for deal_id {deal_in_db.deal_id}.") print(f"✅ No update or upload required for deal_id {deal_in_db.deal_id}.")
@ -188,202 +185,6 @@ class HubspotDataToDb:
session.refresh(new_record) session.refresh(new_record)
return new_record return new_record
def _deprecated_diff(
self,
deal_in_db: HubspotDealData,
hs_deal: Dict[str, str],
hs_company_id: Optional[str],
hs_listing: Optional[Dict[str, str]],
):
def soft_assert(condition: bool, message: str = "Assertion Failed"):
if not condition:
print(f"⚠️ Soft Assert Failed: {message}")
return False
return True
print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})")
# Soft compare key fields
checks = [
soft_assert(
deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"
),
soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"),
soft_assert(
deal_in_db.listing_id == hs_listing.get("listing_id"),
"listing_id mismatch",
),
soft_assert(
deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"),
"landlord_property_id mismatch",
),
soft_assert(
deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"
),
soft_assert(
deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"
),
soft_assert(
deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"
),
soft_assert(
deal_in_db.project_code == hs_deal.get("project_code"),
"project_code mismatch",
),
soft_assert(
deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"
),
soft_assert(
deal_in_db.outcome_notes == hs_deal.get("outcome_notes"),
"outcome_notes mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_description
== hs_deal.get("major_condition_issue_description"),
"major condition description mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_photos
== hs_deal.get("major_condition_issue_photos"),
"major condition issue photos mismatch",
),
soft_assert(
deal_in_db.coordination_status
== hs_deal.get("coordination_status__stage_1_"),
"coordination stage 1 status mismatch",
),
soft_assert(
deal_in_db.coordination_comments
== hs_deal.get("coordination_comments"),
"coordination_comments mismatch",
),
soft_assert(
deal_in_db.design_status == hs_deal.get("retrofit_design_status"),
"retrofit design mismatch",
),
soft_assert(
deal_in_db.pashub_link == hs_deal.get("pashub_link"),
"pashub_link mismatch",
),
soft_assert(
deal_in_db.sharepoint_link == hs_deal.get("sharepoint_link"),
"sharepoint_link mismatch",
),
soft_assert(
deal_in_db.dampmould_growth == hs_deal.get("dampmould_growth"),
"dampmould_growth mismatch",
),
soft_assert(
deal_in_db.damp_mould_and_repairs_comments
== hs_deal.get("damp_mould_and_repairs_comments"),
"damp_mould_and_repairs_comments mismatch",
),
soft_assert(
deal_in_db.pre_sap == hs_deal.get("pre_sap"),
"pre_sap mismatch",
),
soft_assert(
deal_in_db.coordinator == hs_deal.get("coordinator"),
"coordinator mismatch",
),
soft_assert(
deal_in_db.mtp_completion_date
== self._parse_hs_date(hs_deal.get("mtp_completion_date")),
"mtp_completion_date mismatch",
),
soft_assert(
deal_in_db.mtp_re_model_completion_date
== self._parse_hs_date(hs_deal.get("mtp_re_model_completion_date")),
"mtp_re_model_completion_date mismatch",
),
soft_assert(
deal_in_db.ioe_v3_completion_date
== self._parse_hs_date(hs_deal.get("ioe_v3_completion_date")),
"ioe_v3_completion_date mismatch",
),
soft_assert(
deal_in_db.proposed_measures == hs_deal.get("proposed_measures"),
"proposed_measures mismatch",
),
soft_assert(
deal_in_db.approved_package == hs_deal.get("approved_package"),
"approved_package mismatch",
),
soft_assert(
deal_in_db.designer == hs_deal.get("designer"),
"designer mismatch",
),
soft_assert(
deal_in_db.design_completion_date
== self._parse_hs_date(hs_deal.get("design_completion_date")),
"design_completion_date mismatch",
),
soft_assert(
deal_in_db.actual_measures_installed
== hs_deal.get("actual_measures_installed"),
"actual_measures_installed mismatch",
),
soft_assert(
deal_in_db.installer == hs_deal.get("installer"),
"installer mismatch",
),
soft_assert(
deal_in_db.installer_handover == hs_deal.get("installer_handover"),
"installer_handover mismatch",
),
soft_assert(
deal_in_db.lodgement_status == hs_deal.get("lodgement_status"),
"lodgement_status mismatch",
),
soft_assert(
deal_in_db.measures_lodgement_date
== self._parse_hs_date(hs_deal.get("measures_lodgement_date")),
"measures_lodgement_date mismatch",
),
soft_assert(
deal_in_db.lodgement_date
== self._parse_hs_date(hs_deal.get("lodgement_date")),
"lodgement_date mismatch",
),
soft_assert(
deal_in_db.expected_commencement_date
== self._parse_hs_date(hs_deal.get("expected_commencement_date")),
"expected_commencement_date mismatch",
),
soft_assert(
deal_in_db.surveyor == hs_deal.get("surveyor"),
"surveyor mismatch",
),
soft_assert(
deal_in_db.confirmed_survey_date
== self._parse_hs_date(hs_deal.get("confirmed_survey_date")),
"confirmed_survey_date mismatch",
),
soft_assert(
deal_in_db.confirmed_survey_time
== hs_deal.get("confirmed_survey_time"),
"confirmed_survey_time mismatch",
),
soft_assert(
deal_in_db.surveyed_date
== self._parse_hs_date(hs_deal.get("surveyed_date")),
"surveyed_date mismatch",
),
soft_assert(
deal_in_db.design_type == hs_deal.get("design_type"),
"design_type mismatch",
),
]
# If discrepancies found, update from HubSpot
if not all(checks):
print(
f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot."
)
return False
return True
def _update_existing_deal( def _update_existing_deal(
self, self,
existing: HubspotDealData, existing: HubspotDealData,
@ -420,38 +221,36 @@ class HubspotDataToDb:
), ),
"pre_sap": deal_data.get("pre_sap"), "pre_sap": deal_data.get("pre_sap"),
"coordinator": deal_data.get("coordinator"), "coordinator": deal_data.get("coordinator"),
"mtp_completion_date": self._parse_hs_date( "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")),
deal_data.get("mtp_completion_date") "mtp_re_model_completion_date": parse_hs_date(
),
"mtp_re_model_completion_date": self._parse_hs_date(
deal_data.get("mtp_re_model_completion_date") deal_data.get("mtp_re_model_completion_date")
), ),
"ioe_v3_completion_date": self._parse_hs_date( "ioe_v3_completion_date": parse_hs_date(
deal_data.get("ioe_v3_completion_date") deal_data.get("ioe_v3_completion_date")
), ),
"proposed_measures": deal_data.get("proposed_measures"), "proposed_measures": deal_data.get("proposed_measures"),
"approved_package": deal_data.get("approved_package"), "approved_package": deal_data.get("approved_package"),
"designer": deal_data.get("designer"), "designer": deal_data.get("designer"),
"design_completion_date": self._parse_hs_date( "design_completion_date": parse_hs_date(
deal_data.get("design_completion_date") deal_data.get("design_completion_date")
), ),
"actual_measures_installed": deal_data.get("actual_measures_installed"), "actual_measures_installed": deal_data.get("actual_measures_installed"),
"installer": deal_data.get("installer"), "installer": deal_data.get("installer"),
"installer_handover": deal_data.get("installer_handover"), "installer_handover": deal_data.get("installer_handover"),
"lodgement_status": deal_data.get("lodgement_status"), "lodgement_status": deal_data.get("lodgement_status"),
"measures_lodgement_date": self._parse_hs_date( "measures_lodgement_date": parse_hs_date(
deal_data.get("measures_lodgement_date") deal_data.get("measures_lodgement_date")
), ),
"lodgement_date": self._parse_hs_date(deal_data.get("lodgement_date")), "lodgement_date": parse_hs_date(deal_data.get("lodgement_date")),
"expected_commencement_date": self._parse_hs_date( "expected_commencement_date": parse_hs_date(
deal_data.get("expected_commencement_date") deal_data.get("expected_commencement_date")
), ),
"surveyor": deal_data.get("surveyor"), "surveyor": deal_data.get("surveyor"),
"confirmed_survey_date": self._parse_hs_date( "confirmed_survey_date": parse_hs_date(
deal_data.get("confirmed_survey_date") deal_data.get("confirmed_survey_date")
), ),
"confirmed_survey_time": deal_data.get("confirmed_survey_time"), "confirmed_survey_time": deal_data.get("confirmed_survey_time"),
"surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")), "surveyed_date": parse_hs_date(deal_data.get("surveyed_date")),
"design_type": deal_data.get("design_type"), "design_type": deal_data.get("design_type"),
}.items(): }.items():
setattr(existing, attr, value or getattr(existing, attr)) setattr(existing, attr, value or getattr(existing, attr))
@ -491,38 +290,34 @@ class HubspotDataToDb:
), ),
pre_sap=deal_data.get("pre_sap"), pre_sap=deal_data.get("pre_sap"),
coordinator=deal_data.get("coordinator"), coordinator=deal_data.get("coordinator"),
mtp_completion_date=self._parse_hs_date( mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")),
deal_data.get("mtp_completion_date") mtp_re_model_completion_date=parse_hs_date(
),
mtp_re_model_completion_date=self._parse_hs_date(
deal_data.get("mtp_re_model_completion_date") deal_data.get("mtp_re_model_completion_date")
), ),
ioe_v3_completion_date=self._parse_hs_date( ioe_v3_completion_date=parse_hs_date(
deal_data.get("ioe_v3_completion_date") deal_data.get("ioe_v3_completion_date")
), ),
proposed_measures=deal_data.get("proposed_measures"), proposed_measures=deal_data.get("proposed_measures"),
approved_package=deal_data.get("approved_package"), approved_package=deal_data.get("approved_package"),
designer=deal_data.get("designer"), designer=deal_data.get("designer"),
design_completion_date=self._parse_hs_date( design_completion_date=parse_hs_date(
deal_data.get("design_completion_date") deal_data.get("design_completion_date")
), ),
actual_measures_installed=deal_data.get("actual_measures_installed"), actual_measures_installed=deal_data.get("actual_measures_installed"),
installer=deal_data.get("installer"), installer=deal_data.get("installer"),
installer_handover=deal_data.get("installer_handover"), installer_handover=deal_data.get("installer_handover"),
lodgement_status=deal_data.get("lodgement_status"), lodgement_status=deal_data.get("lodgement_status"),
measures_lodgement_date=self._parse_hs_date( measures_lodgement_date=parse_hs_date(
deal_data.get("measures_lodgement_date") deal_data.get("measures_lodgement_date")
), ),
lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")), lodgement_date=parse_hs_date(deal_data.get("lodgement_date")),
expected_commencement_date=self._parse_hs_date( expected_commencement_date=parse_hs_date(
deal_data.get("expected_commencement_date") deal_data.get("expected_commencement_date")
), ),
surveyor=deal_data.get("surveyor"), surveyor=deal_data.get("surveyor"),
confirmed_survey_date=self._parse_hs_date( confirmed_survey_date=parse_hs_date(deal_data.get("confirmed_survey_date")),
deal_data.get("confirmed_survey_date")
),
confirmed_survey_time=deal_data.get("confirmed_survey_time"), confirmed_survey_time=deal_data.get("confirmed_survey_time"),
surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")), surveyed_date=parse_hs_date(deal_data.get("surveyed_date")),
design_type=deal_data.get("design_type"), design_type=deal_data.get("design_type"),
) )

90
etl/hubspot/hubspot_deal_differ.py
View file

@ -1,6 +1,7 @@
from typing import Dict, List, Optional from typing import Dict, List, Optional
from backend.app.db.models.hubspot_deal_data import HubspotDealData from backend.app.db.models.hubspot_deal_data import HubspotDealData
from etl.hubspot.utils import parse_hs_date
class HubspotDealDiffer: class HubspotDealDiffer:
@ -18,7 +19,94 @@ class HubspotDealDiffer:
new_listing: Optional[Dict[str, str]], new_listing: Optional[Dict[str, str]],
old_deal: HubspotDealData, old_deal: HubspotDealData,
) -> bool: ) -> bool:
raise NotImplementedError """
Returns True if ANY difference exists between HubSpot data and DB.
Returns False if everything matches (i.e. no update needed).
"""
# --- Deal ID ---
if str(old_deal.deal_id) != str(new_deal.get("hs_object_id")):
return True
# --- Company ---
if new_company is not None:
if old_deal.company_id != new_company:
return True
# --- Listing ---
hs_listing = new_listing or {}
if old_deal.listing_id != hs_listing.get("listing_id"):
return True
if old_deal.landlord_property_id != hs_listing.get("owner_property_id"):
return True
if old_deal.uprn != hs_listing.get("national_uprn"):
return True
# --- Field mappings ---
FIELD_MAP = {
"outcome": "outcome",
"dealstage": "dealstage",
"dealname": "dealname",
"project_code": "project_code",
"outcome_notes": "outcome_notes",
"major_condition_issue_description": "major_condition_issue_description",
"major_condition_issue_photos": "major_condition_issue_photos",
"coordination_status__stage_1_": "coordination_status",
"coordination_comments": "coordination_comments",
"retrofit_design_status": "design_status",
"pashub_link": "pashub_link",
"sharepoint_link": "sharepoint_link",
"dampmould_growth": "dampmould_growth",
"damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments",
"pre_sap": "pre_sap",
"coordinator": "coordinator",
"proposed_measures": "proposed_measures",
"approved_package": "approved_package",
"designer": "designer",
"actual_measures_installed": "actual_measures_installed",
"installer": "installer",
"installer_handover": "installer_handover",
"lodgement_status": "lodgement_status",
"design_type": "design_type",
"surveyor": "surveyor",
}
for hs_field, db_field in FIELD_MAP.items():
old_value = getattr(old_deal, db_field)
new_value = new_deal.get(hs_field)
if old_value != new_value:
return True
# --- Date fields ---
date_fields = [
("mtp_completion_date", "mtp_completion_date"),
("mtp_re_model_completion_date", "mtp_re_model_completion_date"),
("ioe_v3_completion_date", "ioe_v3_completion_date"),
("design_completion_date", "design_completion_date"),
("measures_lodgement_date", "measures_lodgement_date"),
("lodgement_date", "lodgement_date"),
("expected_commencement_date", "expected_commencement_date"),
("confirmed_survey_date", "confirmed_survey_date"),
("surveyed_date", "surveyed_date"),
]
for hs_field, db_field in date_fields:
old_value = getattr(old_deal, db_field)
new_value = parse_hs_date(new_deal.get(hs_field))
if old_value != new_value:
return True
# --- Time field ---
if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"):
return True
# No differences found
return False
@staticmethod @staticmethod
def check_for_pashub_trigger( def check_for_pashub_trigger(

18
etl/hubspot/scripts/scraper/main.py
View file

@ -63,21 +63,3 @@ def handler(body: dict[str, Any], context: Any) -> None:
): ):
# TODO: trigger pashub file fetcher # TODO: trigger pashub file fetcher
return return
# if db_deal:
# db_client.update_deal_with_checks(db_deal, hubspot_client)
# else:
# hubspot_deal: Dict[str, str]
# company: Optional[str]
# listing: Optional[dict[str, str]]
# hubspot_deal, company, listing = (
# hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
# )
# if company:
# company_data: CompanyData = hubspot_client.get_company_information(company)
# db_client: HubspotDataToDb = HubspotDataToDb()
# db_client.upsert_company(company_data)
# db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)

11
etl/hubspot/utils.py Normal file
View file

@ -0,0 +1,11 @@
from datetime import datetime
from typing import Optional
def parse_hs_date(value: Optional[str]) -> Optional[datetime]:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None