From c439d5f55794f8b1c8f9041d568f28d44f43d0fd Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 9 Apr 2026 09:41:43 +0000 Subject: [PATCH] move everything to etl/hubspot/ --- .../hubspot_trigger_orchestrator/handler.py | 61 -------------- .../hubspot}/hubspot_deal_differ.py | 0 ...ot_trigger_orchestrator_trigger_request.py | 0 etl/hubspot/scripts/scraper/main.py | 80 +++++++++++++------ .../tests/test_hubspot_deal_differ.py | 2 +- 5 files changed, 57 insertions(+), 86 deletions(-) delete mode 100644 backend/hubspot_trigger_orchestrator/handler.py rename {backend/hubspot_trigger_orchestrator => etl/hubspot}/hubspot_deal_differ.py (100%) rename {backend/hubspot_trigger_orchestrator => etl/hubspot}/hubspot_trigger_orchestrator_trigger_request.py (100%) rename {backend/hubspot_trigger_orchestrator => etl/hubspot}/tests/test_hubspot_deal_differ.py (98%) diff --git a/backend/hubspot_trigger_orchestrator/handler.py b/backend/hubspot_trigger_orchestrator/handler.py deleted file mode 100644 index 38724812..00000000 --- a/backend/hubspot_trigger_orchestrator/handler.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from typing import Any, Dict, Mapping, Optional - -from backend.app.db.models.organisation import HubspotDealData -from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer -from backend.hubspot_trigger_orchestrator.hubspot_trigger_orchestrator_trigger_request import ( - HubspotTriggerOrchestratorTriggerRequest, -) -from backend.utils.subtasks import task_handler -from etl.hubspot.hubspotClient import HubspotClient -from etl.hubspot.hubspotDataTodB import HubspotDataToDb -from utils.logger import setup_logger - -logger = setup_logger() - - -@task_handler() -def handler(event: Mapping[str, Any], context: Any) -> None: - - db_client = HubspotDataToDb() - hubspot_client = HubspotClient() - - for record in event.get("Records", []): - body_dict = json.loads(record["body"]) - - logger.debug("Validating request body") - payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body_dict) - logger.debug("Successfully validated request body") - - hubspot_deal_id: str = payload.hubspot_deal_id - - db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id( - hubspot_deal_id - ) - if not db_deal: - # new hubspot deal, no diffing to do - # TODO: trigger hubspot to db ETL - return - - hubspot_deal: Dict[str, str] - company: Optional[str] - listing: Optional[dict[str, str]] - - hubspot_deal, company, listing = ( - hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id) - ) - - if HubspotDealDiffer.check_for_pashub_trigger( - new_deal=hubspot_deal, old_deal=db_deal - ): - # TODO: trigger pashub file fetcher - return - - if HubspotDealDiffer.check_for_db_update_trigger( - new_deal=hubspot_deal, - new_company=company, - new_listing=listing, - old_deal=db_deal, - ): - # TODO: trigger db upsert - return diff --git a/backend/hubspot_trigger_orchestrator/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py similarity index 100% rename from backend/hubspot_trigger_orchestrator/hubspot_deal_differ.py rename to etl/hubspot/hubspot_deal_differ.py diff --git a/backend/hubspot_trigger_orchestrator/hubspot_trigger_orchestrator_trigger_request.py b/etl/hubspot/hubspot_trigger_orchestrator_trigger_request.py similarity index 100% rename from backend/hubspot_trigger_orchestrator/hubspot_trigger_orchestrator_trigger_request.py rename to etl/hubspot/hubspot_trigger_orchestrator_trigger_request.py diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index d8d4a357..8c4af1a7 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -1,39 +1,71 @@ from backend.app.db.models.organisation import HubspotDealData from etl.hubspot.hubspotClient import HubspotClient -from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb + +# from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb +from etl.hubspot.hubspotDataTodB import HubspotDataToDb from backend.utils.subtasks import task_handler from typing import Any, Dict, Optional +from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer +from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( + HubspotTriggerOrchestratorTriggerRequest, +) + @task_handler() def handler(body: dict[str, Any], context: Any) -> None: - hubspot_deal_id = body.get("hubspot_deal_id", "") - - if hubspot_deal_id == "": - raise RuntimeError( - "Missing Hubspot Deal ID in SQS body request, 'hubspot_deal_id'" - ) - hubspot_deal_id = "327170793707" - - hubspot_client = HubspotClient() db_client = HubspotDataToDb() + hubspot_client = HubspotClient() + + payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body) + hubspot_deal_id: str = payload.hubspot_deal_id + db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id( hubspot_deal_id ) - if db_deal: - db_client.update_deal_with_checks(db_deal, hubspot_client) - else: - hubspot_deal: Dict[str, str] - company: Optional[str] - listing: Optional[dict[str, str]] - hubspot_deal, company, listing = ( - hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id) - ) + if not db_deal: + # New hubspot deal, no diffing to do + # TODO: Trigger hubspot to db ETL + return - if company: - company_data: CompanyData = hubspot_client.get_company_information(company) - db_client: HubspotDataToDb = HubspotDataToDb() - db_client.upsert_company(company_data) + hubspot_deal: Dict[str, str] + company: Optional[str] + listing: Optional[dict[str, str]] - db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) + hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing( + hubspot_deal_id + ) + + if HubspotDealDiffer.check_for_pashub_trigger( + new_deal=hubspot_deal, old_deal=db_deal + ): + # TODO: trigger pashub file fetcher + return + + if HubspotDealDiffer.check_for_db_update_trigger( + new_deal=hubspot_deal, + new_company=company, + new_listing=listing, + old_deal=db_deal, + ): + # TODO: trigger db upsert + return + + # if db_deal: + # db_client.update_deal_with_checks(db_deal, hubspot_client) + # else: + # hubspot_deal: Dict[str, str] + # company: Optional[str] + # listing: Optional[dict[str, str]] + + # hubspot_deal, company, listing = ( + # hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id) + # ) + + # if company: + # company_data: CompanyData = hubspot_client.get_company_information(company) + # db_client: HubspotDataToDb = HubspotDataToDb() + # db_client.upsert_company(company_data) + + # db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) diff --git a/backend/hubspot_trigger_orchestrator/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py similarity index 98% rename from backend/hubspot_trigger_orchestrator/tests/test_hubspot_deal_differ.py rename to etl/hubspot/tests/test_hubspot_deal_differ.py index 75fa7927..12c5a288 100644 --- a/backend/hubspot_trigger_orchestrator/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -5,7 +5,7 @@ import uuid import pytest from backend.app.db.models.organisation import HubspotDealData -from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer +from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer BASE_TIME = datetime(2025, 12, 1, 12, 0, 0)