junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move everything to etl/hubspot/

Browse source
This commit is contained in:
Daniel Roth 2026-04-09 09:41:43 +00:00
parent 8ce7619044
commit c439d5f557
5 changed files with 57 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
backend/hubspot_trigger_orchestrator/handler.py
View file

@ -1,61 +0,0 @@
import json
from typing import Any, Dict, Mapping, Optional
from backend.app.db.models.organisation import HubspotDealData
from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer
from backend.hubspot_trigger_orchestrator.hubspot_trigger_orchestrator_trigger_request import (
HubspotTriggerOrchestratorTriggerRequest,
)
from backend.utils.subtasks import task_handler
from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.hubspotDataTodB import HubspotDataToDb
from utils.logger import setup_logger
logger = setup_logger()
@task_handler()
def handler(event: Mapping[str, Any], context: Any) -> None:
db_client = HubspotDataToDb()
hubspot_client = HubspotClient()
for record in event.get("Records", []):
body_dict = json.loads(record["body"])
logger.debug("Validating request body")
payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body_dict)
logger.debug("Successfully validated request body")
hubspot_deal_id: str = payload.hubspot_deal_id
db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id(
hubspot_deal_id
)
if not db_deal:
# new hubspot deal, no diffing to do
# TODO: trigger hubspot to db ETL
return
hubspot_deal: Dict[str, str]
company: Optional[str]
listing: Optional[dict[str, str]]
hubspot_deal, company, listing = (
hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
)
if HubspotDealDiffer.check_for_pashub_trigger(
new_deal=hubspot_deal, old_deal=db_deal
):
# TODO: trigger pashub file fetcher
return
if HubspotDealDiffer.check_for_db_update_trigger(
new_deal=hubspot_deal,
new_company=company,
new_listing=listing,
old_deal=db_deal,
):
# TODO: trigger db upsert
return

0
backend/hubspot_trigger_orchestrator/hubspot_deal_differ.py → etl/hubspot/hubspot_deal_differ.py
View file

0
backend/hubspot_trigger_orchestrator/hubspot_trigger_orchestrator_trigger_request.py → etl/hubspot/hubspot_trigger_orchestrator_trigger_request.py
View file

80
etl/hubspot/scripts/scraper/main.py
View file

@ -1,39 +1,71 @@
from backend.app.db.models.organisation import HubspotDealData
from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
# from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb
from etl.hubspot.hubspotDataTodB import HubspotDataToDb
from backend.utils.subtasks import task_handler
from typing import Any, Dict, Optional
from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import (
HubspotTriggerOrchestratorTriggerRequest,
)
@task_handler()
def handler(body: dict[str, Any], context: Any) -> None:
hubspot_deal_id = body.get("hubspot_deal_id", "")
if hubspot_deal_id == "":
raise RuntimeError(
"Missing Hubspot Deal ID in SQS body request, 'hubspot_deal_id'"
)
hubspot_deal_id = "327170793707"
hubspot_client = HubspotClient()
db_client = HubspotDataToDb()
hubspot_client = HubspotClient()
payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body)
hubspot_deal_id: str = payload.hubspot_deal_id
db_deal: Optional[HubspotDealData] = db_client.find_deal_with_deal_id(
hubspot_deal_id
)
if db_deal:
db_client.update_deal_with_checks(db_deal, hubspot_client)
else:
hubspot_deal: Dict[str, str]
company: Optional[str]
listing: Optional[dict[str, str]]
hubspot_deal, company, listing = (
hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
)
if not db_deal:
# New hubspot deal, no diffing to do
# TODO: Trigger hubspot to db ETL
return
if company:
company_data: CompanyData = hubspot_client.get_company_information(company)
db_client: HubspotDataToDb = HubspotDataToDb()
db_client.upsert_company(company_data)
hubspot_deal: Dict[str, str]
company: Optional[str]
listing: Optional[dict[str, str]]
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
hubspot_deal, company, listing = hubspot_client.get_deal_and_company_and_listing(
hubspot_deal_id
)
if HubspotDealDiffer.check_for_pashub_trigger(
new_deal=hubspot_deal, old_deal=db_deal
):
# TODO: trigger pashub file fetcher
return
if HubspotDealDiffer.check_for_db_update_trigger(
new_deal=hubspot_deal,
new_company=company,
new_listing=listing,
old_deal=db_deal,
):
# TODO: trigger db upsert
return
# if db_deal:
# db_client.update_deal_with_checks(db_deal, hubspot_client)
# else:
# hubspot_deal: Dict[str, str]
# company: Optional[str]
# listing: Optional[dict[str, str]]
# hubspot_deal, company, listing = (
# hubspot_client.get_deal_and_company_and_listing(hubspot_deal_id)
# )
# if company:
# company_data: CompanyData = hubspot_client.get_company_information(company)
# db_client: HubspotDataToDb = HubspotDataToDb()
# db_client.upsert_company(company_data)
# db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)

2
backend/hubspot_trigger_orchestrator/tests/test_hubspot_deal_differ.py → etl/hubspot/tests/test_hubspot_deal_differ.py
View file

@ -5,7 +5,7 @@ import uuid
import pytest
from backend.app.db.models.organisation import HubspotDealData
from backend.hubspot_trigger_orchestrator.hubspot_deal_differ import HubspotDealDiffer
from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer
BASE_TIME = datetime(2025, 12, 1, 12, 0, 0)