diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index e7008618..f0beeee8 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -366,109 +366,9 @@ class HubspotDataToDb: if existing: print(f"🔄 Updating existing deal (deal_id={deal_id})") + self._update_existing_deal(existing, deal_data, listing, company) - for attr, value in { - "dealname": deal_data.get("dealname"), - "dealstage": deal_data.get("dealstage"), - "listing_id": listing.get("listing_id", None) if listing else None, - "landlord_property_id": ( - listing.get("owner_property_id", None) if listing else None - ), - "uprn": listing.get("national_uprn", None) if listing else None, - "outcome": deal_data.get("outcome"), - "outcome_notes": deal_data.get("outcome_notes"), - "project_code": deal_data.get("project_code"), - "company_id": company, - "major_condition_issue_description": deal_data.get( - "major_condition_issue_description" - ), - "major_condition_issue_photos": deal_data.get( - "major_condition_issue_photos" - ), - "coordination_status": deal_data.get( - "coordination_status__stage_1_" - ), - "design_status": deal_data.get("retrofit_design_status"), - "pashub_link": deal_data.get("pashub_link"), - "sharepoint_link": deal_data.get("sharepoint_link"), - "dampmould_growth": deal_data.get("dampmould_growth"), - "damp_mould_and_repairs_comments": deal_data.get( - "damp_mould_and_repairs_comments" - ), - "pre_sap": deal_data.get("pre_sap"), - "coordinator": deal_data.get("coordinator"), - "mtp_completion_date": self._parse_hs_date( - deal_data.get("mtp_completion_date") - ), - "mtp_re_model_completion_date": self._parse_hs_date( - deal_data.get("mtp_re_model_completion_date") - ), - "ioe_v3_completion_date": self._parse_hs_date( - deal_data.get("ioe_v3_completion_date") - ), - "proposed_measures": deal_data.get("proposed_measures"), - "approved_package": deal_data.get("approved_package"), - "designer": deal_data.get("designer"), - "design_completion_date": self._parse_hs_date( - deal_data.get("design_completion_date") - ), - "actual_measures_installed": deal_data.get( - "actual_measures_installed" - ), - "installer": deal_data.get("installer"), - "installer_handover": deal_data.get("installer_handover"), - "lodgement_status": deal_data.get("lodgement_status"), - "measures_lodgement_date": self._parse_hs_date( - deal_data.get("measures_lodgement_date") - ), - "lodgement_date": self._parse_hs_date( - deal_data.get("lodgement_date") - ), - "expected_commencement_date": self._parse_hs_date( - deal_data.get("expected_commencement_date") - ), - "surveyor": deal_data.get("surveyor"), - "confirmed_survey_date": self._parse_hs_date( - deal_data.get("confirmed_survey_date") - ), - "confirmed_survey_time": deal_data.get("confirmed_survey_time"), - "surveyed_date": self._parse_hs_date( - deal_data.get("surveyed_date") - ), - "design_type": deal_data.get("design_type"), - }.items(): - setattr(existing, attr, value or getattr(existing, attr)) - - # Upload if photo exists but S3 link missing - if ( - existing.major_condition_issue_photos - and not existing.major_condition_issue_evidence_s3_url - ): - # Fetch fresh URL from HubSpot instead of using potentially expired stored URL - fresh_deal = hubspot_client.from_deal_id_get_info(existing.deal_id) - photo_url = fresh_deal.get("major_condition_issue_photos") - - if photo_url: - try: - local_file = hubspot_client.download_file_from_url( - photo_url - ) - s3_url = self.s3.upload_file( - local_file, - "retrofit-data-dev", - prefix="hubspot/awaabs_law_evidence/", - ) - existing.major_condition_issue_evidence_s3_url = s3_url - except Exception as e: - print( - f"⚠️ Failed to download photo for deal_id {existing.deal_id}: {e}" - ) - # Continue without the file — don't crash the update - finally: - if "local_file" in locals() and os.path.exists(local_file): - os.remove(local_file) - else: - print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}") + self._handle_existing_photo_upload(existing, hubspot_client) session.add(existing) session.commit() @@ -477,94 +377,207 @@ class HubspotDataToDb: else: print(f"🆕 Inserting new deal (deal_id={deal_id})") - new_record = HubspotDealData( - deal_id=deal_id, - dealname=deal_data.get("dealname"), - dealstage=deal_data.get("dealstage"), - listing_id=listing.get("listing_id", None) if listing else None, - landlord_property_id=( - listing.get("owner_property_id") if listing else None - ), - uprn=listing.get("national_uprn") if listing else None, - outcome=deal_data.get("outcome"), - outcome_notes=deal_data.get("outcome_notes"), - project_code=deal_data.get("project_code"), - company_id=company, - major_condition_issue_description=deal_data.get( - "major_condition_issue_description" - ), - major_condition_issue_photos=deal_data.get( - "major_condition_issue_photos" - ), - coordination_status=deal_data.get("coordination_status__stage_1_"), - design_status=deal_data.get("retrofit_design_status"), - pashub_link=deal_data.get("pashub_link"), - sharepoint_link=deal_data.get("sharepoint_link"), - dampmould_growth=deal_data.get("dampmould_growth"), - damp_mould_and_repairs_comments=deal_data.get( - "damp_mould_and_repairs_comments" - ), - pre_sap=deal_data.get("pre_sap"), - coordinator=deal_data.get("coordinator"), - mtp_completion_date=self._parse_hs_date( - deal_data.get("mtp_completion_date") - ), - mtp_re_model_completion_date=self._parse_hs_date( - deal_data.get("mtp_re_model_completion_date") - ), - ioe_v3_completion_date=self._parse_hs_date( - deal_data.get("ioe_v3_completion_date") - ), - proposed_measures=deal_data.get("proposed_measures"), - approved_package=deal_data.get("approved_package"), - designer=deal_data.get("designer"), - design_completion_date=self._parse_hs_date( - deal_data.get("design_completion_date") - ), - actual_measures_installed=deal_data.get( - "actual_measures_installed" - ), - installer=deal_data.get("installer"), - installer_handover=deal_data.get("installer_handover"), - lodgement_status=deal_data.get("lodgement_status"), - measures_lodgement_date=self._parse_hs_date( - deal_data.get("measures_lodgement_date") - ), - lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")), - expected_commencement_date=self._parse_hs_date( - deal_data.get("expected_commencement_date") - ), - surveyor=deal_data.get("surveyor"), - confirmed_survey_date=self._parse_hs_date( - deal_data.get("confirmed_survey_date") - ), - confirmed_survey_time=deal_data.get("confirmed_survey_time"), - surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")), - design_type=deal_data.get("design_type"), + new_record: HubspotDealData = self._build_new_deal( + deal_id, deal_data, listing, company ) # Handle upload at insert time - if new_record.major_condition_issue_photos: - try: - local_file = hubspot_client.download_file_from_url( - new_record.major_condition_issue_photos - ) - s3_url = self.s3.upload_file( - local_file, - "retrofit-data-dev", - prefix="hubspot/awaabs_law_evidence/", - ) - new_record.major_condition_issue_evidence_s3_url = s3_url - except Exception as e: - print( - f"⚠️ Failed to download photo for deal_id {new_record.deal_id}: {e}" - ) - # Continue without the file — don't crash the insert - finally: - if "local_file" in locals() and os.path.exists(local_file): - os.remove(local_file) + self._handle_new_photo_upload(new_record, hubspot_client) session.add(new_record) session.commit() session.refresh(new_record) return new_record + + def _update_existing_deal( + self, + existing: HubspotDealData, + deal_data: Dict[str, str], + listing: Optional[dict[str, str]], + company: Optional[str], + ): + for attr, value in { + "dealname": deal_data.get("dealname"), + "dealstage": deal_data.get("dealstage"), + "listing_id": listing.get("listing_id", None) if listing else None, + "landlord_property_id": ( + listing.get("owner_property_id", None) if listing else None + ), + "uprn": listing.get("national_uprn", None) if listing else None, + "outcome": deal_data.get("outcome"), + "outcome_notes": deal_data.get("outcome_notes"), + "project_code": deal_data.get("project_code"), + "company_id": company, + "major_condition_issue_description": deal_data.get( + "major_condition_issue_description" + ), + "major_condition_issue_photos": deal_data.get( + "major_condition_issue_photos" + ), + "coordination_status": deal_data.get("coordination_status__stage_1_"), + "design_status": deal_data.get("retrofit_design_status"), + "pashub_link": deal_data.get("pashub_link"), + "sharepoint_link": deal_data.get("sharepoint_link"), + "dampmould_growth": deal_data.get("dampmould_growth"), + "damp_mould_and_repairs_comments": deal_data.get( + "damp_mould_and_repairs_comments" + ), + "pre_sap": deal_data.get("pre_sap"), + "coordinator": deal_data.get("coordinator"), + "mtp_completion_date": self._parse_hs_date( + deal_data.get("mtp_completion_date") + ), + "mtp_re_model_completion_date": self._parse_hs_date( + deal_data.get("mtp_re_model_completion_date") + ), + "ioe_v3_completion_date": self._parse_hs_date( + deal_data.get("ioe_v3_completion_date") + ), + "proposed_measures": deal_data.get("proposed_measures"), + "approved_package": deal_data.get("approved_package"), + "designer": deal_data.get("designer"), + "design_completion_date": self._parse_hs_date( + deal_data.get("design_completion_date") + ), + "actual_measures_installed": deal_data.get("actual_measures_installed"), + "installer": deal_data.get("installer"), + "installer_handover": deal_data.get("installer_handover"), + "lodgement_status": deal_data.get("lodgement_status"), + "measures_lodgement_date": self._parse_hs_date( + deal_data.get("measures_lodgement_date") + ), + "lodgement_date": self._parse_hs_date(deal_data.get("lodgement_date")), + "expected_commencement_date": self._parse_hs_date( + deal_data.get("expected_commencement_date") + ), + "surveyor": deal_data.get("surveyor"), + "confirmed_survey_date": self._parse_hs_date( + deal_data.get("confirmed_survey_date") + ), + "confirmed_survey_time": deal_data.get("confirmed_survey_time"), + "surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")), + "design_type": deal_data.get("design_type"), + }.items(): + setattr(existing, attr, value or getattr(existing, attr)) + + def _build_new_deal( + self, + deal_id: str, + deal_data: Dict[str, str], + listing: Optional[dict[str, str]], + company: Optional[str], + ) -> HubspotDealData: + return HubspotDealData( + deal_id=deal_id, + dealname=deal_data.get("dealname"), + dealstage=deal_data.get("dealstage"), + listing_id=listing.get("listing_id") if listing else None, + landlord_property_id=( + listing.get("owner_property_id") if listing else None + ), + uprn=listing.get("national_uprn") if listing else None, + outcome=deal_data.get("outcome"), + outcome_notes=deal_data.get("outcome_notes"), + project_code=deal_data.get("project_code"), + company_id=company, + major_condition_issue_description=deal_data.get( + "major_condition_issue_description" + ), + major_condition_issue_photos=deal_data.get("major_condition_issue_photos"), + coordination_status=deal_data.get("coordination_status__stage_1_"), + design_status=deal_data.get("retrofit_design_status"), + pashub_link=deal_data.get("pashub_link"), + sharepoint_link=deal_data.get("sharepoint_link"), + dampmould_growth=deal_data.get("dampmould_growth"), + damp_mould_and_repairs_comments=deal_data.get( + "damp_mould_and_repairs_comments" + ), + pre_sap=deal_data.get("pre_sap"), + coordinator=deal_data.get("coordinator"), + mtp_completion_date=self._parse_hs_date( + deal_data.get("mtp_completion_date") + ), + mtp_re_model_completion_date=self._parse_hs_date( + deal_data.get("mtp_re_model_completion_date") + ), + ioe_v3_completion_date=self._parse_hs_date( + deal_data.get("ioe_v3_completion_date") + ), + proposed_measures=deal_data.get("proposed_measures"), + approved_package=deal_data.get("approved_package"), + designer=deal_data.get("designer"), + design_completion_date=self._parse_hs_date( + deal_data.get("design_completion_date") + ), + actual_measures_installed=deal_data.get("actual_measures_installed"), + installer=deal_data.get("installer"), + installer_handover=deal_data.get("installer_handover"), + lodgement_status=deal_data.get("lodgement_status"), + measures_lodgement_date=self._parse_hs_date( + deal_data.get("measures_lodgement_date") + ), + lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")), + expected_commencement_date=self._parse_hs_date( + deal_data.get("expected_commencement_date") + ), + surveyor=deal_data.get("surveyor"), + confirmed_survey_date=self._parse_hs_date( + deal_data.get("confirmed_survey_date") + ), + confirmed_survey_time=deal_data.get("confirmed_survey_time"), + surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")), + design_type=deal_data.get("design_type"), + ) + + def _handle_existing_photo_upload( + self, + existing: HubspotDealData, + hubspot_client: HubspotClient, + ): + if ( + existing.major_condition_issue_photos + and not existing.major_condition_issue_evidence_s3_url + ): + fresh_deal = hubspot_client.from_deal_id_get_info(existing.deal_id) + photo_url = fresh_deal.get("major_condition_issue_photos") + + if not photo_url: + print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}") + return + + self._upload_photo_to_s3(existing, photo_url, hubspot_client) + + def _handle_new_photo_upload( + self, + record: HubspotDealData, + hubspot_client: HubspotClient, + ): + if record.major_condition_issue_photos: + self._upload_photo_to_s3( + record, + record.major_condition_issue_photos, + hubspot_client, + ) + + def _upload_photo_to_s3( + self, + record: HubspotDealData, + photo_url: str, + hubspot_client: HubspotClient, + ): + try: + local_file = hubspot_client.download_file_from_url(photo_url) + + s3_url = self.s3.upload_file( + local_file, + "retrofit-data-dev", + prefix="hubspot/awaabs_law_evidence/", + ) + + record.major_condition_issue_evidence_s3_url = s3_url + + except Exception as e: + print(f"⚠️ Failed to upload photo for deal_id {record.deal_id}: {e}") + finally: + if "local_file" in locals() and os.path.exists(local_file): + os.remove(local_file)