move everything to etl/hubspot/

backend/hubspot_trigger_orchestrator/handler.py

@@ -1,61 +0,0 @@
-import json
-from typing import Any, Dict, Mapping, Optional
-
-from backend.app.db.models.organisation import HubspotDealData
-from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer
-from backend.hubspot_trigger_orchestrator.hubspot_trigger_orchestrator_trigger_request import (
-    HubspotTriggerOrchestratorTriggerRequest,
-)
-from backend.utils.subtasks import task_handler
-from etl.hubspot.hubspotClient import HubspotClient
-from etl.hubspot.hubspotDataTodB import HubspotDataToDb
-from utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-@task_handler()
-def handler(event: Mapping[str, Any], context: Any) -> None:
-
-    db_client = HubspotDataToDb()
-    hubspot_client = HubspotClient()
-
-    for record in event.get("Records", []):
-        body_dict = json.loads(record["body"])
-
-        logger.debug("Validating request body")
-        payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body_dict)
-        logger.debug("Successfully validated request body")
-
-        hubspot_deal_id: str = payload.hubspot_deal_id
-
-        db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id(
-            hubspot_deal_id
-        )
-        if not db_deal:
-            # new hubspot deal, no diffing to do
-            # TODO: trigger hubspot to db ETL
-            return
-
-        hubspot_deal: Dict[str, str]
-        company: Optional[str]
-        listing: Optional[dict[str, str]]
-
-        hubspot_deal, company, listing = (
-            hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
-        )
-
-        if HubspotDealDiffer.check_for_pashub_trigger(
-            new_deal=hubspot_deal, old_deal=db_deal
-        ):
-            # TODO: trigger pashub file fetcher
-            return
-
-        if HubspotDealDiffer.check_for_db_update_trigger(
-            new_deal=hubspot_deal,
-            new_company=company,
-            new_listing=listing,
-            old_deal=db_deal,
-        ):
-            # TODO: trigger db upsert
-            return

etl/hubspot/scripts/scraper/main.py

@@ -1,39 +1,71 @@
 from backend.app.db.models.organisation import HubspotDealData
 from etl.hubspot.hubspotClient import HubspotClient
-from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
+
+# from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
+from etl.hubspot.hubspotDataTodB import HubspotDataToDb
 from backend.utils.subtasks import task_handler
 from typing import Any, Dict, Optional
 
+from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
+from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import (
+    HubspotTriggerOrchestratorTriggerRequest,
+)
+
 
 @task_handler()
 def handler(body: dict[str, Any], context: Any) -> None:
-    hubspot_deal_id = body.get("hubspot_deal_id", "")
-
-    if hubspot_deal_id == "":
-        raise RuntimeError(
-            "Missing Hubspot Deal ID in SQS body request, 'hubspot_deal_id'"
-        )
-        hubspot_deal_id = "327170793707"
-
-    hubspot_client = HubspotClient()
     db_client = HubspotDataToDb()
+    hubspot_client = HubspotClient()
+
+    payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body)
+    hubspot_deal_id: str = payload.hubspot_deal_id
+
     db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id(
         hubspot_deal_id
     )
-    if db_deal:
-        db_client.update_deal_with_checks(db_deal, hubspot_client)
-    else:
-        hubspot_deal: Dict[str, str]
-        company: Optional[str]
-        listing: Optional[dict[str, str]]
 
-        hubspot_deal, company, listing = (
+    if not db_deal:
-            hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
+        # New hubspot deal, no diffing to do
-        )
+        # TODO: Trigger hubspot to db ETL
+        return
 
-        if company:
+    hubspot_deal: Dict[str, str]
-            company_data: CompanyData = hubspot_client.get_company_information(company)
+    company: Optional[str]
-            db_client: HubspotDataToDb = HubspotDataToDb()
+    listing: Optional[dict[str, str]]
-            db_client.upsert_company(company_data)
 
-        db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
+    hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing(
+        hubspot_deal_id
+    )
+
+    if HubspotDealDiffer.check_for_pashub_trigger(
+        new_deal=hubspot_deal, old_deal=db_deal
+    ):
+        # TODO: trigger pashub file fetcher
+        return
+
+    if HubspotDealDiffer.check_for_db_update_trigger(
+        new_deal=hubspot_deal,
+        new_company=company,
+        new_listing=listing,
+        old_deal=db_deal,
+    ):
+        # TODO: trigger db upsert
+        return
+
+    # if db_deal:
+    #     db_client.update_deal_with_checks(db_deal, hubspot_client)
+    # else:
+    #     hubspot_deal: Dict[str, str]
+    #     company: Optional[str]
+    #     listing: Optional[dict[str, str]]
+
+    #     hubspot_deal, company, listing = (
+    #         hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
+    #     )
+
+    #     if company:
+    #         company_data: CompanyData = hubspot_client.get_company_information(company)
+    #         db_client: HubspotDataToDb = HubspotDataToDb()
+    #         db_client.upsert_company(company_data)
+
+    #     db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)

etl/hubspot/tests/test_hubspot_deal_differ.py

@@ -5,7 +5,7 @@ import uuid
 import pytest
 
 from backend.app.db.models.organisation import HubspotDealData
-from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer
+from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
 
 
 BASE_TIME = datetime(2025, 12, 1, 12, 0, 0)