From 3873da710910426c0315f1ed3b422d8c4409945c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 20 Feb 2026 12:26:10 +0000 Subject: [PATCH] add soft checks --- etl/db/hubSpotLoad.py | 143 +++++++++++++++++++++++++++++++++--------- 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/etl/db/hubSpotLoad.py b/etl/db/hubSpotLoad.py index fad4741..7e435e9 100644 --- a/etl/db/hubSpotLoad.py +++ b/etl/db/hubSpotLoad.py @@ -49,6 +49,7 @@ class HubspotTodb: Checks if a deal needs updating and syncs it with HubSpot. Also handles major_condition_issue_photos file upload to S3 with integrity check. """ + def soft_assert(condition, message="Assertion Failed"): if not condition: print(f"⚠️ Soft Assert Failed: {message}") @@ -57,38 +58,89 @@ class HubspotTodb: print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})") - hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db(deal_in_db.deal_id) + hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db( + deal_in_db.deal_id + ) # Soft compare key fields checks = [ - soft_assert(deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"), + soft_assert( + deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch" + ), soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"), - soft_assert(deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"), "landlord_property_id mismatch"), - soft_assert(deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"), - soft_assert(deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"), - soft_assert(deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"), - soft_assert(deal_in_db.project_code == hs_deal.get("project_code"), "project_code mismatch"), - soft_assert(deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"), - soft_assert(deal_in_db.outcome_notes == hs_deal.get("outcome_notes"), "outcome_notes mismatch"), - soft_assert(deal_in_db.major_condition_issue_description == hs_deal.get("major_condition_issue_description"), "major condition description mismatch"), - soft_assert(deal_in_db.major_condition_issue_photos == hs_deal.get("major_condition_issue_photos"), "major condition issue photos mismatch"), + soft_assert( + deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"), + "landlord_property_id mismatch", + ), + soft_assert( + deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch" + ), + soft_assert( + deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch" + ), + soft_assert( + deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch" + ), + soft_assert( + deal_in_db.project_code == hs_deal.get("project_code"), + "project_code mismatch", + ), + soft_assert( + deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch" + ), + soft_assert( + deal_in_db.outcome_notes == hs_deal.get("outcome_notes"), + "outcome_notes mismatch", + ), + soft_assert( + deal_in_db.major_condition_issue_description + == hs_deal.get("major_condition_issue_description"), + "major condition description mismatch", + ), + soft_assert( + deal_in_db.major_condition_issue_photos + == hs_deal.get("major_condition_issue_photos"), + "major condition issue photos mismatch", + ), + soft_assert( + deal_in_db.coordination_status + == hs_listing.get("coordination_status__stage_1_"), + "uprn mismatch", + ), + soft_assert( + deal_in_db.design_status == hs_deal.get("retrofit_design_status"), + "outcome_notes mismatch", + ), ] # If discrepancies found, update from HubSpot if not all(checks): - print(f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot.") - return self.upsert_hubspot_deal(hs_deal, hs_company_id, hs_listing, hubspot_client) + print( + f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot." + ) + return self.upsert_hubspot_deal( + hs_deal, hs_company_id, hs_listing, hubspot_client + ) # Handle photo upload if it exists but S3 URL is missing - if deal_in_db.major_condition_issue_photos and not deal_in_db.major_condition_issue_evidence_s3_url: - print(f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3...") + if ( + deal_in_db.major_condition_issue_photos + and not deal_in_db.major_condition_issue_evidence_s3_url + ): + print( + f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3..." + ) # Download from HubSpot - local_file = hubspot_client.download_file_from_url(deal_in_db.major_condition_issue_photos) + local_file = hubspot_client.download_file_from_url( + deal_in_db.major_condition_issue_photos + ) # Upload to S3 bucket = "retrofit-data-dev" - s3_url = self.s3.upload_file(local_file, bucket, prefix="hubspot/awaabs_law_evidence/") + s3_url = self.s3.upload_file( + local_file, bucket, prefix="hubspot/awaabs_law_evidence/" + ) # Download again to verify integrity downloaded = self.s3.download_from_url(s3_url) @@ -121,7 +173,9 @@ class HubspotTodb: print(deal_data) deal_id = deal_data.get("hs_object_id") - statement = select(HubspotDealData).where(HubspotDealData.deal_id == deal_id) + statement = select(HubspotDealData).where( + HubspotDealData.deal_id == deal_id + ) existing = session.exec(statement).first() if existing: @@ -136,19 +190,38 @@ class HubspotTodb: "outcome_notes": deal_data.get("outcome_notes"), "project_code": deal_data.get("project_code"), "company_id": company, - "major_condition_issue_description": deal_data.get("major_condition_issue_description"), - "major_condition_issue_photos": deal_data.get("major_condition_issue_photos"), - "major_condition_issue_description": deal_data.get("major_condition_issue_description"), - "major_condition_issue_photos": deal_data.get("major_condition_issue_photos"), - "coordination_status": deal_data.get("coordination_status__stage_1_"), + "major_condition_issue_description": deal_data.get( + "major_condition_issue_description" + ), + "major_condition_issue_photos": deal_data.get( + "major_condition_issue_photos" + ), + "major_condition_issue_description": deal_data.get( + "major_condition_issue_description" + ), + "major_condition_issue_photos": deal_data.get( + "major_condition_issue_photos" + ), + "coordination_status": deal_data.get( + "coordination_status__stage_1_" + ), "design_status": deal_data.get("retrofit_design_status"), }.items(): setattr(existing, attr, value or getattr(existing, attr)) # Upload if photo exists but S3 link missing - if existing.major_condition_issue_photos and not existing.major_condition_issue_evidence_s3_url: - local_file = hubspot_client.download_file_from_url(existing.major_condition_issue_photos) - s3_url = self.s3.upload_file(local_file, "retrofit-data-dev", prefix="hubspot/awaabs_law_evidence/") + if ( + existing.major_condition_issue_photos + and not existing.major_condition_issue_evidence_s3_url + ): + local_file = hubspot_client.download_file_from_url( + existing.major_condition_issue_photos + ) + s3_url = self.s3.upload_file( + local_file, + "retrofit-data-dev", + prefix="hubspot/awaabs_law_evidence/", + ) existing.major_condition_issue_evidence_s3_url = s3_url session.add(existing) @@ -168,16 +241,26 @@ class HubspotTodb: outcome_notes=deal_data.get("outcome_notes"), project_code=deal_data.get("project_code"), company_id=company, - major_condition_issue_description=deal_data.get("major_condition_issue_description"), - major_condition_issue_photos=deal_data.get("major_condition_issue_photos"), + major_condition_issue_description=deal_data.get( + "major_condition_issue_description" + ), + major_condition_issue_photos=deal_data.get( + "major_condition_issue_photos" + ), coordination_status=deal_data.get("coordination_status__stage_1_"), design_status=deal_data.get("retrofit_design_status"), ) # Handle upload at insert time if new_record.major_condition_issue_photos: - local_file = hubspot_client.download_file_from_url(new_record.major_condition_issue_photos) - s3_url = self.s3.upload_file(local_file, "retrofit-data-dev", prefix="hubspot/awaabs_law_evidence/") + local_file = hubspot_client.download_file_from_url( + new_record.major_condition_issue_photos + ) + s3_url = self.s3.upload_file( + local_file, + "retrofit-data-dev", + prefix="hubspot/awaabs_law_evidence/", + ) new_record.major_condition_issue_evidence_s3_url = s3_url session.add(new_record)