diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index d5a51ace..1d7607e0 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -59,19 +59,21 @@ class HubspotDealData(SQLModel, table=True): surveyed_date: Optional[datetime] = Field(default=None) design_type: Optional[str] = Field(default=None) - created_at: datetime = Field( + created_at: Optional[datetime] = Field( sa_column=Column( DateTime(timezone=True), server_default=text("(NOW() AT TIME ZONE 'utc')"), nullable=False, - ) + ), + default=None, # Nullable in db but optional here as value is set on db save for new record ) - updated_at: datetime = Field( + updated_at: Optional[datetime] = Field( sa_column=Column( DateTime(timezone=True), server_default=text("(NOW() AT TIME ZONE 'utc')"), onupdate=func.now(), nullable=False, - ) + ), + default=None, # Nullable in db but optional here as value is set on db save for new record ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 5ebc8c73..210c9593 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -10,6 +10,7 @@ from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.s3_uploader import S3Uploader from backend.app.db.connection import db_read_session from backend.app.db.models.organisation import Organisation +from etl.hubspot.utils import parse_hs_date class HubspotDataToDb: @@ -60,11 +61,7 @@ class HubspotDataToDb: session.commit() return record - def new_record_to_hubspot_data(self, deal_data, company, listing, hubspot_client): - print("⚠️ Deprecated — use the new interface instead.") - return self.upsert_deal(deal_data, company, listing, hubspot_client) - - def find_all_deals_with_company_id(self, company_id): + def find_all_deals_with_company_id(self, company_id: str): """Returns a list of deals for a given company_id.""" with db_read_session() as session: return ( @@ -137,7 +134,7 @@ class HubspotDataToDb: return False else: - print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}") + print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}") else: print(f"✅ No update or upload required for deal_id {deal_in_db.deal_id}.") @@ -188,202 +185,6 @@ class HubspotDataToDb: session.refresh(new_record) return new_record - def _deprecated_diff( - self, - deal_in_db: HubspotDealData, - hs_deal: Dict[str, str], - hs_company_id: Optional[str], - hs_listing: Optional[Dict[str, str]], - ): - def soft_assert(condition: bool, message: str = "Assertion Failed"): - if not condition: - print(f"⚠️ Soft Assert Failed: {message}") - return False - return True - - print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})") - - # Soft compare key fields - checks = [ - soft_assert( - deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch" - ), - soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"), - soft_assert( - deal_in_db.listing_id == hs_listing.get("listing_id"), - "listing_id mismatch", - ), - soft_assert( - deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"), - "landlord_property_id mismatch", - ), - soft_assert( - deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch" - ), - soft_assert( - deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch" - ), - soft_assert( - deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch" - ), - soft_assert( - deal_in_db.project_code == hs_deal.get("project_code"), - "project_code mismatch", - ), - soft_assert( - deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch" - ), - soft_assert( - deal_in_db.outcome_notes == hs_deal.get("outcome_notes"), - "outcome_notes mismatch", - ), - soft_assert( - deal_in_db.major_condition_issue_description - == hs_deal.get("major_condition_issue_description"), - "major condition description mismatch", - ), - soft_assert( - deal_in_db.major_condition_issue_photos - == hs_deal.get("major_condition_issue_photos"), - "major condition issue photos mismatch", - ), - soft_assert( - deal_in_db.coordination_status - == hs_deal.get("coordination_status__stage_1_"), - "coordination stage 1 status mismatch", - ), - soft_assert( - deal_in_db.coordination_comments - == hs_deal.get("coordination_comments"), - "coordination_comments mismatch", - ), - soft_assert( - deal_in_db.design_status == hs_deal.get("retrofit_design_status"), - "retrofit design mismatch", - ), - soft_assert( - deal_in_db.pashub_link == hs_deal.get("pashub_link"), - "pashub_link mismatch", - ), - soft_assert( - deal_in_db.sharepoint_link == hs_deal.get("sharepoint_link"), - "sharepoint_link mismatch", - ), - soft_assert( - deal_in_db.dampmould_growth == hs_deal.get("dampmould_growth"), - "dampmould_growth mismatch", - ), - soft_assert( - deal_in_db.damp_mould_and_repairs_comments - == hs_deal.get("damp_mould_and_repairs_comments"), - "damp_mould_and_repairs_comments mismatch", - ), - soft_assert( - deal_in_db.pre_sap == hs_deal.get("pre_sap"), - "pre_sap mismatch", - ), - soft_assert( - deal_in_db.coordinator == hs_deal.get("coordinator"), - "coordinator mismatch", - ), - soft_assert( - deal_in_db.mtp_completion_date - == self._parse_hs_date(hs_deal.get("mtp_completion_date")), - "mtp_completion_date mismatch", - ), - soft_assert( - deal_in_db.mtp_re_model_completion_date - == self._parse_hs_date(hs_deal.get("mtp_re_model_completion_date")), - "mtp_re_model_completion_date mismatch", - ), - soft_assert( - deal_in_db.ioe_v3_completion_date - == self._parse_hs_date(hs_deal.get("ioe_v3_completion_date")), - "ioe_v3_completion_date mismatch", - ), - soft_assert( - deal_in_db.proposed_measures == hs_deal.get("proposed_measures"), - "proposed_measures mismatch", - ), - soft_assert( - deal_in_db.approved_package == hs_deal.get("approved_package"), - "approved_package mismatch", - ), - soft_assert( - deal_in_db.designer == hs_deal.get("designer"), - "designer mismatch", - ), - soft_assert( - deal_in_db.design_completion_date - == self._parse_hs_date(hs_deal.get("design_completion_date")), - "design_completion_date mismatch", - ), - soft_assert( - deal_in_db.actual_measures_installed - == hs_deal.get("actual_measures_installed"), - "actual_measures_installed mismatch", - ), - soft_assert( - deal_in_db.installer == hs_deal.get("installer"), - "installer mismatch", - ), - soft_assert( - deal_in_db.installer_handover == hs_deal.get("installer_handover"), - "installer_handover mismatch", - ), - soft_assert( - deal_in_db.lodgement_status == hs_deal.get("lodgement_status"), - "lodgement_status mismatch", - ), - soft_assert( - deal_in_db.measures_lodgement_date - == self._parse_hs_date(hs_deal.get("measures_lodgement_date")), - "measures_lodgement_date mismatch", - ), - soft_assert( - deal_in_db.lodgement_date - == self._parse_hs_date(hs_deal.get("lodgement_date")), - "lodgement_date mismatch", - ), - soft_assert( - deal_in_db.expected_commencement_date - == self._parse_hs_date(hs_deal.get("expected_commencement_date")), - "expected_commencement_date mismatch", - ), - soft_assert( - deal_in_db.surveyor == hs_deal.get("surveyor"), - "surveyor mismatch", - ), - soft_assert( - deal_in_db.confirmed_survey_date - == self._parse_hs_date(hs_deal.get("confirmed_survey_date")), - "confirmed_survey_date mismatch", - ), - soft_assert( - deal_in_db.confirmed_survey_time - == hs_deal.get("confirmed_survey_time"), - "confirmed_survey_time mismatch", - ), - soft_assert( - deal_in_db.surveyed_date - == self._parse_hs_date(hs_deal.get("surveyed_date")), - "surveyed_date mismatch", - ), - soft_assert( - deal_in_db.design_type == hs_deal.get("design_type"), - "design_type mismatch", - ), - ] - - # If discrepancies found, update from HubSpot - if not all(checks): - print( - f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot." - ) - return False - - return True - def _update_existing_deal( self, existing: HubspotDealData, @@ -420,38 +221,36 @@ class HubspotDataToDb: ), "pre_sap": deal_data.get("pre_sap"), "coordinator": deal_data.get("coordinator"), - "mtp_completion_date": self._parse_hs_date( - deal_data.get("mtp_completion_date") - ), - "mtp_re_model_completion_date": self._parse_hs_date( + "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), + "mtp_re_model_completion_date": parse_hs_date( deal_data.get("mtp_re_model_completion_date") ), - "ioe_v3_completion_date": self._parse_hs_date( + "ioe_v3_completion_date": parse_hs_date( deal_data.get("ioe_v3_completion_date") ), "proposed_measures": deal_data.get("proposed_measures"), "approved_package": deal_data.get("approved_package"), "designer": deal_data.get("designer"), - "design_completion_date": self._parse_hs_date( + "design_completion_date": parse_hs_date( deal_data.get("design_completion_date") ), "actual_measures_installed": deal_data.get("actual_measures_installed"), "installer": deal_data.get("installer"), "installer_handover": deal_data.get("installer_handover"), "lodgement_status": deal_data.get("lodgement_status"), - "measures_lodgement_date": self._parse_hs_date( + "measures_lodgement_date": parse_hs_date( deal_data.get("measures_lodgement_date") ), - "lodgement_date": self._parse_hs_date(deal_data.get("lodgement_date")), - "expected_commencement_date": self._parse_hs_date( + "lodgement_date": parse_hs_date(deal_data.get("lodgement_date")), + "expected_commencement_date": parse_hs_date( deal_data.get("expected_commencement_date") ), "surveyor": deal_data.get("surveyor"), - "confirmed_survey_date": self._parse_hs_date( + "confirmed_survey_date": parse_hs_date( deal_data.get("confirmed_survey_date") ), "confirmed_survey_time": deal_data.get("confirmed_survey_time"), - "surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")), + "surveyed_date": parse_hs_date(deal_data.get("surveyed_date")), "design_type": deal_data.get("design_type"), }.items(): setattr(existing, attr, value or getattr(existing, attr)) @@ -491,38 +290,34 @@ class HubspotDataToDb: ), pre_sap=deal_data.get("pre_sap"), coordinator=deal_data.get("coordinator"), - mtp_completion_date=self._parse_hs_date( - deal_data.get("mtp_completion_date") - ), - mtp_re_model_completion_date=self._parse_hs_date( + mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), + mtp_re_model_completion_date=parse_hs_date( deal_data.get("mtp_re_model_completion_date") ), - ioe_v3_completion_date=self._parse_hs_date( + ioe_v3_completion_date=parse_hs_date( deal_data.get("ioe_v3_completion_date") ), proposed_measures=deal_data.get("proposed_measures"), approved_package=deal_data.get("approved_package"), designer=deal_data.get("designer"), - design_completion_date=self._parse_hs_date( + design_completion_date=parse_hs_date( deal_data.get("design_completion_date") ), actual_measures_installed=deal_data.get("actual_measures_installed"), installer=deal_data.get("installer"), installer_handover=deal_data.get("installer_handover"), lodgement_status=deal_data.get("lodgement_status"), - measures_lodgement_date=self._parse_hs_date( + measures_lodgement_date=parse_hs_date( deal_data.get("measures_lodgement_date") ), - lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")), - expected_commencement_date=self._parse_hs_date( + lodgement_date=parse_hs_date(deal_data.get("lodgement_date")), + expected_commencement_date=parse_hs_date( deal_data.get("expected_commencement_date") ), surveyor=deal_data.get("surveyor"), - confirmed_survey_date=self._parse_hs_date( - deal_data.get("confirmed_survey_date") - ), + confirmed_survey_date=parse_hs_date(deal_data.get("confirmed_survey_date")), confirmed_survey_time=deal_data.get("confirmed_survey_time"), - surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")), + surveyed_date=parse_hs_date(deal_data.get("surveyed_date")), design_type=deal_data.get("design_type"), ) diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index dd992243..42def3b2 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData +from etl.hubspot.utils import parse_hs_date class HubspotDealDiffer: @@ -18,7 +19,94 @@ class HubspotDealDiffer: new_listing: Optional[Dict[str, str]], old_deal: HubspotDealData, ) -> bool: - raise NotImplementedError + """ + Returns True if ANY difference exists between HubSpot data and DB. + Returns False if everything matches (i.e. no update needed). + """ + + # --- Deal ID --- + if str(old_deal.deal_id) != str(new_deal.get("hs_object_id")): + return True + + # --- Company --- + if new_company is not None: + if old_deal.company_id != new_company: + return True + + # --- Listing --- + hs_listing = new_listing or {} + + if old_deal.listing_id != hs_listing.get("listing_id"): + return True + + if old_deal.landlord_property_id != hs_listing.get("owner_property_id"): + return True + + if old_deal.uprn != hs_listing.get("national_uprn"): + return True + + # --- Field mappings --- + FIELD_MAP = { + "outcome": "outcome", + "dealstage": "dealstage", + "dealname": "dealname", + "project_code": "project_code", + "outcome_notes": "outcome_notes", + "major_condition_issue_description": "major_condition_issue_description", + "major_condition_issue_photos": "major_condition_issue_photos", + "coordination_status__stage_1_": "coordination_status", + "coordination_comments": "coordination_comments", + "retrofit_design_status": "design_status", + "pashub_link": "pashub_link", + "sharepoint_link": "sharepoint_link", + "dampmould_growth": "dampmould_growth", + "damp_mould_and_repairs_comments": "damp_mould_and_repairs_comments", + "pre_sap": "pre_sap", + "coordinator": "coordinator", + "proposed_measures": "proposed_measures", + "approved_package": "approved_package", + "designer": "designer", + "actual_measures_installed": "actual_measures_installed", + "installer": "installer", + "installer_handover": "installer_handover", + "lodgement_status": "lodgement_status", + "design_type": "design_type", + "surveyor": "surveyor", + } + + for hs_field, db_field in FIELD_MAP.items(): + old_value = getattr(old_deal, db_field) + new_value = new_deal.get(hs_field) + + if old_value != new_value: + return True + + # --- Date fields --- + date_fields = [ + ("mtp_completion_date", "mtp_completion_date"), + ("mtp_re_model_completion_date", "mtp_re_model_completion_date"), + ("ioe_v3_completion_date", "ioe_v3_completion_date"), + ("design_completion_date", "design_completion_date"), + ("measures_lodgement_date", "measures_lodgement_date"), + ("lodgement_date", "lodgement_date"), + ("expected_commencement_date", "expected_commencement_date"), + ("confirmed_survey_date", "confirmed_survey_date"), + ("surveyed_date", "surveyed_date"), + ] + + for hs_field, db_field in date_fields: + old_value = getattr(old_deal, db_field) + new_value = parse_hs_date(new_deal.get(hs_field)) + + if old_value != new_value: + return True + + # --- Time field --- + if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"): + return True + + # No differences found + return False @staticmethod def check_for_pashub_trigger( diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 826d7e05..5d5b2b26 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -63,21 +63,3 @@ def handler(body: dict[str, Any], context: Any) -> None: ): # TODO: trigger pashub file fetcher return - - # if db_deal: - # db_client.update_deal_with_checks(db_deal, hubspot_client) - # else: - # hubspot_deal: Dict[str, str] - # company: Optional[str] - # listing: Optional[dict[str, str]] - - # hubspot_deal, company, listing = ( - # hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id) - # ) - - # if company: - # company_data: CompanyData = hubspot_client.get_company_information(company) - # db_client: HubspotDataToDb = HubspotDataToDb() - # db_client.upsert_company(company_data) - - # db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) diff --git a/etl/hubspot/utils.py b/etl/hubspot/utils.py new file mode 100644 index 00000000..9fbeae62 --- /dev/null +++ b/etl/hubspot/utils.py @@ -0,0 +1,11 @@ +from datetime import datetime +from typing import Optional + + +def parse_hs_date(value: Optional[str]) -> Optional[datetime]: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None