junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #127 from Hestia-Homes/feature/new

Browse source 
add soft checks
This commit is contained in:
Jun-te Kim 2026-02-20 12:27:13 +00:00 • committed by GitHub
commit 28f8d1d823
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 113 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

143
etl/db/hubSpotLoad.py
View file

@ -49,6 +49,7 @@ class HubspotTodb:
Checks if a deal needs updating and syncs it with HubSpot.
Also handles major_condition_issue_photos file upload to S3 with integrity check.
"""
def soft_assert(condition, message="Assertion Failed"):
if not condition:
print(f"⚠️ Soft Assert Failed: {message}")
@ -57,38 +58,89 @@ class HubspotTodb:
print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})")
hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db(deal_in_db.deal_id)
hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db(
deal_in_db.deal_id
)
# Soft compare key fields
checks = [
soft_assert(deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"),
soft_assert(
deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"
),
soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"),
soft_assert(deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"), "landlord_property_id mismatch"),
soft_assert(deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"),
soft_assert(deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"),
soft_assert(deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"),
soft_assert(deal_in_db.project_code == hs_deal.get("project_code"), "project_code mismatch"),
soft_assert(deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"),
soft_assert(deal_in_db.outcome_notes == hs_deal.get("outcome_notes"), "outcome_notes mismatch"),
soft_assert(deal_in_db.major_condition_issue_description == hs_deal.get("major_condition_issue_description"), "major condition description mismatch"),
soft_assert(deal_in_db.major_condition_issue_photos == hs_deal.get("major_condition_issue_photos"), "major condition issue photos mismatch"),
soft_assert(
deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"),
"landlord_property_id mismatch",
),
soft_assert(
deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"
),
soft_assert(
deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"
),
soft_assert(
deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"
),
soft_assert(
deal_in_db.project_code == hs_deal.get("project_code"),
"project_code mismatch",
),
soft_assert(
deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"
),
soft_assert(
deal_in_db.outcome_notes == hs_deal.get("outcome_notes"),
"outcome_notes mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_description
== hs_deal.get("major_condition_issue_description"),
"major condition description mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_photos
== hs_deal.get("major_condition_issue_photos"),
"major condition issue photos mismatch",
),
soft_assert(
deal_in_db.coordination_status
== hs_listing.get("coordination_status__stage_1_"),
"uprn mismatch",
),
soft_assert(
deal_in_db.design_status == hs_deal.get("retrofit_design_status"),
"outcome_notes mismatch",
),
]
# If discrepancies found, update from HubSpot
if not all(checks):
print(f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot.")
return self.upsert_hubspot_deal(hs_deal, hs_company_id, hs_listing, hubspot_client)
print(
f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot."
)
return self.upsert_hubspot_deal(
hs_deal, hs_company_id, hs_listing, hubspot_client
)
# Handle photo upload if it exists but S3 URL is missing
if deal_in_db.major_condition_issue_photos and not deal_in_db.major_condition_issue_evidence_s3_url:
print(f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3...")
if (
deal_in_db.major_condition_issue_photos
and not deal_in_db.major_condition_issue_evidence_s3_url
):
print(
f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3..."
)
# Download from HubSpot
local_file = hubspot_client.download_file_from_url(deal_in_db.major_condition_issue_photos)
local_file = hubspot_client.download_file_from_url(
deal_in_db.major_condition_issue_photos
)
# Upload to S3
bucket = "retrofit-data-dev"
s3_url = self.s3.upload_file(local_file, bucket, prefix="hubspot/awaabs_law_evidence/")
s3_url = self.s3.upload_file(
local_file, bucket, prefix="hubspot/awaabs_law_evidence/"
)
# Download again to verify integrity
downloaded = self.s3.download_from_url(s3_url)
@ -121,7 +173,9 @@ class HubspotTodb:
print(deal_data)
deal_id = deal_data.get("hs_object_id")
statement = select(HubspotDealData).where(HubspotDealData.deal_id == deal_id)
statement = select(HubspotDealData).where(
HubspotDealData.deal_id == deal_id
)
existing = session.exec(statement).first()
if existing:
@ -136,19 +190,38 @@ class HubspotTodb:
"outcome_notes": deal_data.get("outcome_notes"),
"project_code": deal_data.get("project_code"),
"company_id": company,
"major_condition_issue_description": deal_data.get("major_condition_issue_description"),
"major_condition_issue_photos": deal_data.get("major_condition_issue_photos"),
"major_condition_issue_description": deal_data.get("major_condition_issue_description"),
"major_condition_issue_photos": deal_data.get("major_condition_issue_photos"),
"coordination_status": deal_data.get("coordination_status__stage_1_"),
"major_condition_issue_description": deal_data.get(
"major_condition_issue_description"
),
"major_condition_issue_photos": deal_data.get(
"major_condition_issue_photos"
),
"major_condition_issue_description": deal_data.get(
"major_condition_issue_description"
),
"major_condition_issue_photos": deal_data.get(
"major_condition_issue_photos"
),
"coordination_status": deal_data.get(
"coordination_status__stage_1_"
),
"design_status": deal_data.get("retrofit_design_status"),
}.items():
setattr(existing, attr, value or getattr(existing, attr))
# Upload if photo exists but S3 link missing
if existing.major_condition_issue_photos and not existing.major_condition_issue_evidence_s3_url:
local_file = hubspot_client.download_file_from_url(existing.major_condition_issue_photos)
s3_url = self.s3.upload_file(local_file, "retrofit-data-dev", prefix="hubspot/awaabs_law_evidence/")
if (
existing.major_condition_issue_photos
and not existing.major_condition_issue_evidence_s3_url
):
local_file = hubspot_client.download_file_from_url(
existing.major_condition_issue_photos
)
s3_url = self.s3.upload_file(
local_file,
"retrofit-data-dev",
prefix="hubspot/awaabs_law_evidence/",
)
existing.major_condition_issue_evidence_s3_url = s3_url
session.add(existing)
@ -168,16 +241,26 @@ class HubspotTodb:
outcome_notes=deal_data.get("outcome_notes"),
project_code=deal_data.get("project_code"),
company_id=company,
major_condition_issue_description=deal_data.get("major_condition_issue_description"),
major_condition_issue_photos=deal_data.get("major_condition_issue_photos"),
major_condition_issue_description=deal_data.get(
"major_condition_issue_description"
),
major_condition_issue_photos=deal_data.get(
"major_condition_issue_photos"
),
coordination_status=deal_data.get("coordination_status__stage_1_"),
design_status=deal_data.get("retrofit_design_status"),
)
# Handle upload at insert time
if new_record.major_condition_issue_photos:
local_file = hubspot_client.download_file_from_url(new_record.major_condition_issue_photos)
s3_url = self.s3.upload_file(local_file, "retrofit-data-dev", prefix="hubspot/awaabs_law_evidence/")
local_file = hubspot_client.download_file_from_url(
new_record.major_condition_issue_photos
)
s3_url = self.s3.upload_file(
local_file,
"retrofit-data-dev",
prefix="hubspot/awaabs_law_evidence/",
)
new_record.major_condition_issue_evidence_s3_url = s3_url
session.add(new_record)