mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
refactor upsert_deal by introducing helper methods
This commit is contained in:
Daniel Roth
parent
f719149c03
commit
dd0522713e
1 changed files with 198 additions and 185 deletions
|
|
@ -366,109 +366,9 @@ class HubspotDataToDb:
|
|||
|
||||
if existing:
|
||||
print(f"🔄 Updating existing deal (deal_id={deal_id})")
|
||||
self._update_existing_deal(existing, deal_data, listing, company)
|
||||
|
||||
for attr, value in {
|
||||
"dealname": deal_data.get("dealname"),
|
||||
"dealstage": deal_data.get("dealstage"),
|
||||
"listing_id": listing.get("listing_id", None) if listing else None,
|
||||
"landlord_property_id": (
|
||||
listing.get("owner_property_id", None) if listing else None
|
||||
),
|
||||
"uprn": listing.get("national_uprn", None) if listing else None,
|
||||
"outcome": deal_data.get("outcome"),
|
||||
"outcome_notes": deal_data.get("outcome_notes"),
|
||||
"project_code": deal_data.get("project_code"),
|
||||
"company_id": company,
|
||||
"major_condition_issue_description": deal_data.get(
|
||||
"major_condition_issue_description"
|
||||
),
|
||||
"major_condition_issue_photos": deal_data.get(
|
||||
"major_condition_issue_photos"
|
||||
),
|
||||
"coordination_status": deal_data.get(
|
||||
"coordination_status__stage_1_"
|
||||
),
|
||||
"design_status": deal_data.get("retrofit_design_status"),
|
||||
"pashub_link": deal_data.get("pashub_link"),
|
||||
"sharepoint_link": deal_data.get("sharepoint_link"),
|
||||
"dampmould_growth": deal_data.get("dampmould_growth"),
|
||||
"damp_mould_and_repairs_comments": deal_data.get(
|
||||
"damp_mould_and_repairs_comments"
|
||||
),
|
||||
"pre_sap": deal_data.get("pre_sap"),
|
||||
"coordinator": deal_data.get("coordinator"),
|
||||
"mtp_completion_date": self._parse_hs_date(
|
||||
deal_data.get("mtp_completion_date")
|
||||
),
|
||||
"mtp_re_model_completion_date": self._parse_hs_date(
|
||||
deal_data.get("mtp_re_model_completion_date")
|
||||
),
|
||||
"ioe_v3_completion_date": self._parse_hs_date(
|
||||
deal_data.get("ioe_v3_completion_date")
|
||||
),
|
||||
"proposed_measures": deal_data.get("proposed_measures"),
|
||||
"approved_package": deal_data.get("approved_package"),
|
||||
"designer": deal_data.get("designer"),
|
||||
"design_completion_date": self._parse_hs_date(
|
||||
deal_data.get("design_completion_date")
|
||||
),
|
||||
"actual_measures_installed": deal_data.get(
|
||||
"actual_measures_installed"
|
||||
),
|
||||
"installer": deal_data.get("installer"),
|
||||
"installer_handover": deal_data.get("installer_handover"),
|
||||
"lodgement_status": deal_data.get("lodgement_status"),
|
||||
"measures_lodgement_date": self._parse_hs_date(
|
||||
deal_data.get("measures_lodgement_date")
|
||||
),
|
||||
"lodgement_date": self._parse_hs_date(
|
||||
deal_data.get("lodgement_date")
|
||||
),
|
||||
"expected_commencement_date": self._parse_hs_date(
|
||||
deal_data.get("expected_commencement_date")
|
||||
),
|
||||
"surveyor": deal_data.get("surveyor"),
|
||||
"confirmed_survey_date": self._parse_hs_date(
|
||||
deal_data.get("confirmed_survey_date")
|
||||
),
|
||||
"confirmed_survey_time": deal_data.get("confirmed_survey_time"),
|
||||
"surveyed_date": self._parse_hs_date(
|
||||
deal_data.get("surveyed_date")
|
||||
),
|
||||
"design_type": deal_data.get("design_type"),
|
||||
}.items():
|
||||
setattr(existing, attr, value or getattr(existing, attr))
|
||||
|
||||
# Upload if photo exists but S3 link missing
|
||||
if (
|
||||
existing.major_condition_issue_photos
|
||||
and not existing.major_condition_issue_evidence_s3_url
|
||||
):
|
||||
# Fetch fresh URL from HubSpot instead of using potentially expired stored URL
|
||||
fresh_deal = hubspot_client.from_deal_id_get_info(existing.deal_id)
|
||||
photo_url = fresh_deal.get("major_condition_issue_photos")
|
||||
|
||||
if photo_url:
|
||||
try:
|
||||
local_file = hubspot_client.download_file_from_url(
|
||||
photo_url
|
||||
)
|
||||
s3_url = self.s3.upload_file(
|
||||
local_file,
|
||||
"retrofit-data-dev",
|
||||
prefix="hubspot/awaabs_law_evidence/",
|
||||
)
|
||||
existing.major_condition_issue_evidence_s3_url = s3_url
|
||||
except Exception as e:
|
||||
print(
|
||||
f"⚠️ Failed to download photo for deal_id {existing.deal_id}: {e}"
|
||||
)
|
||||
# Continue without the file — don't crash the update
|
||||
finally:
|
||||
if "local_file" in locals() and os.path.exists(local_file):
|
||||
os.remove(local_file)
|
||||
else:
|
||||
print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}")
|
||||
self._handle_existing_photo_upload(existing, hubspot_client)
|
||||
|
||||
session.add(existing)
|
||||
session.commit()
|
||||
|
|
@ -477,94 +377,207 @@ class HubspotDataToDb:
|
|||
|
||||
else:
|
||||
print(f"🆕 Inserting new deal (deal_id={deal_id})")
|
||||
new_record = HubspotDealData(
|
||||
deal_id=deal_id,
|
||||
dealname=deal_data.get("dealname"),
|
||||
dealstage=deal_data.get("dealstage"),
|
||||
listing_id=listing.get("listing_id", None) if listing else None,
|
||||
landlord_property_id=(
|
||||
listing.get("owner_property_id") if listing else None
|
||||
),
|
||||
uprn=listing.get("national_uprn") if listing else None,
|
||||
outcome=deal_data.get("outcome"),
|
||||
outcome_notes=deal_data.get("outcome_notes"),
|
||||
project_code=deal_data.get("project_code"),
|
||||
company_id=company,
|
||||
major_condition_issue_description=deal_data.get(
|
||||
"major_condition_issue_description"
|
||||
),
|
||||
major_condition_issue_photos=deal_data.get(
|
||||
"major_condition_issue_photos"
|
||||
),
|
||||
coordination_status=deal_data.get("coordination_status__stage_1_"),
|
||||
design_status=deal_data.get("retrofit_design_status"),
|
||||
pashub_link=deal_data.get("pashub_link"),
|
||||
sharepoint_link=deal_data.get("sharepoint_link"),
|
||||
dampmould_growth=deal_data.get("dampmould_growth"),
|
||||
damp_mould_and_repairs_comments=deal_data.get(
|
||||
"damp_mould_and_repairs_comments"
|
||||
),
|
||||
pre_sap=deal_data.get("pre_sap"),
|
||||
coordinator=deal_data.get("coordinator"),
|
||||
mtp_completion_date=self._parse_hs_date(
|
||||
deal_data.get("mtp_completion_date")
|
||||
),
|
||||
mtp_re_model_completion_date=self._parse_hs_date(
|
||||
deal_data.get("mtp_re_model_completion_date")
|
||||
),
|
||||
ioe_v3_completion_date=self._parse_hs_date(
|
||||
deal_data.get("ioe_v3_completion_date")
|
||||
),
|
||||
proposed_measures=deal_data.get("proposed_measures"),
|
||||
approved_package=deal_data.get("approved_package"),
|
||||
designer=deal_data.get("designer"),
|
||||
design_completion_date=self._parse_hs_date(
|
||||
deal_data.get("design_completion_date")
|
||||
),
|
||||
actual_measures_installed=deal_data.get(
|
||||
"actual_measures_installed"
|
||||
),
|
||||
installer=deal_data.get("installer"),
|
||||
installer_handover=deal_data.get("installer_handover"),
|
||||
lodgement_status=deal_data.get("lodgement_status"),
|
||||
measures_lodgement_date=self._parse_hs_date(
|
||||
deal_data.get("measures_lodgement_date")
|
||||
),
|
||||
lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")),
|
||||
expected_commencement_date=self._parse_hs_date(
|
||||
deal_data.get("expected_commencement_date")
|
||||
),
|
||||
surveyor=deal_data.get("surveyor"),
|
||||
confirmed_survey_date=self._parse_hs_date(
|
||||
deal_data.get("confirmed_survey_date")
|
||||
),
|
||||
confirmed_survey_time=deal_data.get("confirmed_survey_time"),
|
||||
surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
|
||||
design_type=deal_data.get("design_type"),
|
||||
new_record: HubspotDealData = self._build_new_deal(
|
||||
deal_id, deal_data, listing, company
|
||||
)
|
||||
|
||||
# Handle upload at insert time
|
||||
if new_record.major_condition_issue_photos:
|
||||
try:
|
||||
local_file = hubspot_client.download_file_from_url(
|
||||
new_record.major_condition_issue_photos
|
||||
)
|
||||
s3_url = self.s3.upload_file(
|
||||
local_file,
|
||||
"retrofit-data-dev",
|
||||
prefix="hubspot/awaabs_law_evidence/",
|
||||
)
|
||||
new_record.major_condition_issue_evidence_s3_url = s3_url
|
||||
except Exception as e:
|
||||
print(
|
||||
f"⚠️ Failed to download photo for deal_id {new_record.deal_id}: {e}"
|
||||
)
|
||||
# Continue without the file — don't crash the insert
|
||||
finally:
|
||||
if "local_file" in locals() and os.path.exists(local_file):
|
||||
os.remove(local_file)
|
||||
self._handle_new_photo_upload(new_record, hubspot_client)
|
||||
|
||||
session.add(new_record)
|
||||
session.commit()
|
||||
session.refresh(new_record)
|
||||
return new_record
|
||||
|
||||
def _update_existing_deal(
|
||||
self,
|
||||
existing: HubspotDealData,
|
||||
deal_data: Dict[str, str],
|
||||
listing: Optional[dict[str, str]],
|
||||
company: Optional[str],
|
||||
):
|
||||
for attr, value in {
|
||||
"dealname": deal_data.get("dealname"),
|
||||
"dealstage": deal_data.get("dealstage"),
|
||||
"listing_id": listing.get("listing_id", None) if listing else None,
|
||||
"landlord_property_id": (
|
||||
listing.get("owner_property_id", None) if listing else None
|
||||
),
|
||||
"uprn": listing.get("national_uprn", None) if listing else None,
|
||||
"outcome": deal_data.get("outcome"),
|
||||
"outcome_notes": deal_data.get("outcome_notes"),
|
||||
"project_code": deal_data.get("project_code"),
|
||||
"company_id": company,
|
||||
"major_condition_issue_description": deal_data.get(
|
||||
"major_condition_issue_description"
|
||||
),
|
||||
"major_condition_issue_photos": deal_data.get(
|
||||
"major_condition_issue_photos"
|
||||
),
|
||||
"coordination_status": deal_data.get("coordination_status__stage_1_"),
|
||||
"design_status": deal_data.get("retrofit_design_status"),
|
||||
"pashub_link": deal_data.get("pashub_link"),
|
||||
"sharepoint_link": deal_data.get("sharepoint_link"),
|
||||
"dampmould_growth": deal_data.get("dampmould_growth"),
|
||||
"damp_mould_and_repairs_comments": deal_data.get(
|
||||
"damp_mould_and_repairs_comments"
|
||||
),
|
||||
"pre_sap": deal_data.get("pre_sap"),
|
||||
"coordinator": deal_data.get("coordinator"),
|
||||
"mtp_completion_date": self._parse_hs_date(
|
||||
deal_data.get("mtp_completion_date")
|
||||
),
|
||||
"mtp_re_model_completion_date": self._parse_hs_date(
|
||||
deal_data.get("mtp_re_model_completion_date")
|
||||
),
|
||||
"ioe_v3_completion_date": self._parse_hs_date(
|
||||
deal_data.get("ioe_v3_completion_date")
|
||||
),
|
||||
"proposed_measures": deal_data.get("proposed_measures"),
|
||||
"approved_package": deal_data.get("approved_package"),
|
||||
"designer": deal_data.get("designer"),
|
||||
"design_completion_date": self._parse_hs_date(
|
||||
deal_data.get("design_completion_date")
|
||||
),
|
||||
"actual_measures_installed": deal_data.get("actual_measures_installed"),
|
||||
"installer": deal_data.get("installer"),
|
||||
"installer_handover": deal_data.get("installer_handover"),
|
||||
"lodgement_status": deal_data.get("lodgement_status"),
|
||||
"measures_lodgement_date": self._parse_hs_date(
|
||||
deal_data.get("measures_lodgement_date")
|
||||
),
|
||||
"lodgement_date": self._parse_hs_date(deal_data.get("lodgement_date")),
|
||||
"expected_commencement_date": self._parse_hs_date(
|
||||
deal_data.get("expected_commencement_date")
|
||||
),
|
||||
"surveyor": deal_data.get("surveyor"),
|
||||
"confirmed_survey_date": self._parse_hs_date(
|
||||
deal_data.get("confirmed_survey_date")
|
||||
),
|
||||
"confirmed_survey_time": deal_data.get("confirmed_survey_time"),
|
||||
"surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")),
|
||||
"design_type": deal_data.get("design_type"),
|
||||
}.items():
|
||||
setattr(existing, attr, value or getattr(existing, attr))
|
||||
|
||||
def _build_new_deal(
|
||||
self,
|
||||
deal_id: str,
|
||||
deal_data: Dict[str, str],
|
||||
listing: Optional[dict[str, str]],
|
||||
company: Optional[str],
|
||||
) -> HubspotDealData:
|
||||
return HubspotDealData(
|
||||
deal_id=deal_id,
|
||||
dealname=deal_data.get("dealname"),
|
||||
dealstage=deal_data.get("dealstage"),
|
||||
listing_id=listing.get("listing_id") if listing else None,
|
||||
landlord_property_id=(
|
||||
listing.get("owner_property_id") if listing else None
|
||||
),
|
||||
uprn=listing.get("national_uprn") if listing else None,
|
||||
outcome=deal_data.get("outcome"),
|
||||
outcome_notes=deal_data.get("outcome_notes"),
|
||||
project_code=deal_data.get("project_code"),
|
||||
company_id=company,
|
||||
major_condition_issue_description=deal_data.get(
|
||||
"major_condition_issue_description"
|
||||
),
|
||||
major_condition_issue_photos=deal_data.get("major_condition_issue_photos"),
|
||||
coordination_status=deal_data.get("coordination_status__stage_1_"),
|
||||
design_status=deal_data.get("retrofit_design_status"),
|
||||
pashub_link=deal_data.get("pashub_link"),
|
||||
sharepoint_link=deal_data.get("sharepoint_link"),
|
||||
dampmould_growth=deal_data.get("dampmould_growth"),
|
||||
damp_mould_and_repairs_comments=deal_data.get(
|
||||
"damp_mould_and_repairs_comments"
|
||||
),
|
||||
pre_sap=deal_data.get("pre_sap"),
|
||||
coordinator=deal_data.get("coordinator"),
|
||||
mtp_completion_date=self._parse_hs_date(
|
||||
deal_data.get("mtp_completion_date")
|
||||
),
|
||||
mtp_re_model_completion_date=self._parse_hs_date(
|
||||
deal_data.get("mtp_re_model_completion_date")
|
||||
),
|
||||
ioe_v3_completion_date=self._parse_hs_date(
|
||||
deal_data.get("ioe_v3_completion_date")
|
||||
),
|
||||
proposed_measures=deal_data.get("proposed_measures"),
|
||||
approved_package=deal_data.get("approved_package"),
|
||||
designer=deal_data.get("designer"),
|
||||
design_completion_date=self._parse_hs_date(
|
||||
deal_data.get("design_completion_date")
|
||||
),
|
||||
actual_measures_installed=deal_data.get("actual_measures_installed"),
|
||||
installer=deal_data.get("installer"),
|
||||
installer_handover=deal_data.get("installer_handover"),
|
||||
lodgement_status=deal_data.get("lodgement_status"),
|
||||
measures_lodgement_date=self._parse_hs_date(
|
||||
deal_data.get("measures_lodgement_date")
|
||||
),
|
||||
lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")),
|
||||
expected_commencement_date=self._parse_hs_date(
|
||||
deal_data.get("expected_commencement_date")
|
||||
),
|
||||
surveyor=deal_data.get("surveyor"),
|
||||
confirmed_survey_date=self._parse_hs_date(
|
||||
deal_data.get("confirmed_survey_date")
|
||||
),
|
||||
confirmed_survey_time=deal_data.get("confirmed_survey_time"),
|
||||
surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
|
||||
design_type=deal_data.get("design_type"),
|
||||
)
|
||||
|
||||
def _handle_existing_photo_upload(
|
||||
self,
|
||||
existing: HubspotDealData,
|
||||
hubspot_client: HubspotClient,
|
||||
):
|
||||
if (
|
||||
existing.major_condition_issue_photos
|
||||
and not existing.major_condition_issue_evidence_s3_url
|
||||
):
|
||||
fresh_deal = hubspot_client.from_deal_id_get_info(existing.deal_id)
|
||||
photo_url = fresh_deal.get("major_condition_issue_photos")
|
||||
|
||||
if not photo_url:
|
||||
print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}")
|
||||
return
|
||||
|
||||
self._upload_photo_to_s3(existing, photo_url, hubspot_client)
|
||||
|
||||
def _handle_new_photo_upload(
|
||||
self,
|
||||
record: HubspotDealData,
|
||||
hubspot_client: HubspotClient,
|
||||
):
|
||||
if record.major_condition_issue_photos:
|
||||
self._upload_photo_to_s3(
|
||||
record,
|
||||
record.major_condition_issue_photos,
|
||||
hubspot_client,
|
||||
)
|
||||
|
||||
def _upload_photo_to_s3(
|
||||
self,
|
||||
record: HubspotDealData,
|
||||
photo_url: str,
|
||||
hubspot_client: HubspotClient,
|
||||
):
|
||||
try:
|
||||
local_file = hubspot_client.download_file_from_url(photo_url)
|
||||
|
||||
s3_url = self.s3.upload_file(
|
||||
local_file,
|
||||
"retrofit-data-dev",
|
||||
prefix="hubspot/awaabs_law_evidence/",
|
||||
)
|
||||
|
||||
record.major_condition_issue_evidence_s3_url = s3_url
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to upload photo for deal_id {record.deal_id}: {e}")
|
||||
finally:
|
||||
if "local_file" in locals() and os.path.exists(local_file):
|
||||
os.remove(local_file)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue