diff --git a/backend/hubspot_trigger_orchestrator/handler.py b/backend/hubspot_trigger_orchestrator/handler.py index 1f83ed80..c79fe2b9 100644 --- a/backend/hubspot_trigger_orchestrator/handler.py +++ b/backend/hubspot_trigger_orchestrator/handler.py @@ -41,7 +41,7 @@ def handler(event: Mapping[str, Any], context: Any) -> None: company: Optional[str] listing: Optional[dict[str, str]] - hubspot_deal, company, listing = hubspot_client.get_deal_info_for_db( + hubspot_deal, company, listing = hubspot_client.get_deal_company_listing( hubspot_deal_id ) diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index a9ea535d..777ad482 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -26,10 +26,10 @@ from hubspot.crm.associations.v4.models import ( # type: ignore[reportMissingTy ForwardPaging as AssociationsPaging, NextPage as AssociationsPagingNext, ) -from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from backend.app.config import get_settings +from etl.hubspot.company_data import CompanyData from utils.logger import setup_logger import mimetypes @@ -279,18 +279,12 @@ class HubspotClient: deal_info: dict[str, str] = cast(dict[str, str], deal.properties) # type: ignore[reportUnknownMemberType] return deal_info - def get_deal_info_for_db( + def get_deal_company_listing( self, deal_id: str ) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]: deal: dict[str, str] = self.from_deal_id_get_info(deal_id) company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id) - - if company: - company_data: CompanyData = self.get_company_information(company) - dbloader: HubspotDataToDb = HubspotDataToDb() - dbloader.upsert_company(company_data) - listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing( deal_id ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 36167bf0..49dd1685 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -2,17 +2,14 @@ from backend.app.db.connection import db_read_session from backend.app.db.models.organisation import Organisation, HubspotDealData from sqlmodel import select from datetime import datetime, timezone -from typing import TypedDict, Optional +from typing import Dict, Optional +from etl.hubspot.company_data import CompanyData +from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.s3_uploader import S3Uploader import hashlib import os -class CompanyData(TypedDict): - hs_object_id: str - name: str - - class HubspotDataToDb: def __init__(self): self.s3 = S3Uploader( @@ -98,7 +95,9 @@ class HubspotDataToDb: sha256.update(chunk) return sha256.hexdigest() - def update_deal_with_checks(self, deal_in_db, hubspot_client) -> bool: + def update_deal_with_checks( + self, deal_in_db: HubspotDealData, hubspot_client: HubspotClient + ) -> bool: """ Checks if a deal needs updating and syncs it with HubSpot. Also handles major_condition_issue_photos file upload to S3 with integrity check. @@ -112,7 +111,7 @@ class HubspotDataToDb: print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})") - hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db( + hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_company_listing( deal_in_db.deal_id ) @@ -346,7 +345,13 @@ class HubspotDataToDb: return True - def upsert_deal(self, deal_data, company, listing, hubspot_client): + def upsert_deal( + self, + deal_data: Dict[str, str], + company: Optional[str], + listing: Optional[dict[str, str]], + hubspot_client: HubspotClient, + ): """ Inserts or updates a deal record. Also uploads photos if present and adds S3 URL. diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index a003ad28..e5658a20 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -1,7 +1,8 @@ +from backend.app.db.models.organisation import HubspotDealData from etl.hubspot.hubspotClient import HubspotClient -from etl.hubspot.hubspotDataTodB import HubspotDataToDb +from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from backend.utils.subtasks import task_handler -from typing import Any +from typing import Any, Dict, Optional @task_handler() @@ -14,11 +15,25 @@ def handler(body: dict[str, Any], context: Any) -> None: ) hubspot_deal_id = "327170793707" - hubspot: HubspotClient = HubspotClient() - dbloader: HubspotDataToDb = HubspotDataToDb() - db_deal = dbloader.find_deal_with_deal_id(hubspot_deal_id) + hubspot_client = HubspotClient() + db_client = HubspotDataToDb() + db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id( + hubspot_deal_id + ) if db_deal: - dbloader.update_deal_with_checks(db_deal, hubspot) + db_client.update_deal_with_checks(db_deal, hubspot_client) else: - hubspot_deal, company, listing = hubspot.get_deal_info_for_db(hubspot_deal_id) - dbloader.upsert_deal(hubspot_deal, company, listing, hubspot) + hubspot_deal: Dict[str, str] + company: Optional[str] + listing: Optional[dict[str, str]] + + hubspot_deal, company, listing = hubspot_client.get_deal_company_listing( + hubspot_deal_id + ) + + if company: + company_data: CompanyData = hubspot_client.get_company_information(company) + db_client: HubspotDataToDb = HubspotDataToDb() + db_client.upsert_company(company_data) + + db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) diff --git a/etl/hubspot/tests/test_hubspot_client_integration.py b/etl/hubspot/tests/test_hubspot_client_integration.py index a3d8ae54..d0dd818a 100644 --- a/etl/hubspot/tests/test_hubspot_client_integration.py +++ b/etl/hubspot/tests/test_hubspot_client_integration.py @@ -71,7 +71,7 @@ class TestHubspotClientIntegration: def test_get_deal_info_for_db(self, client: HubspotClient): deal_id: str = "263490768079" - deal, company, listing = client.get_deal_info_for_db(deal_id) + deal, company, listing = client.get_deal_company_listing(deal_id) assert "dealname" in deal assert "dealstage" in deal