junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove time

Browse source
This commit is contained in:
Jun-te Kim 2026-04-02 10:21:22 +00:00
parent e1ea6f79f9
commit 4c696c3d4c
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
etl/hubspot/scripts/scraper/bulk_load.py
View file

@ -2,7 +2,6 @@ from etl.hubspot.hubspotClient import HubspotClient, Companies, Pipeline
from etl.hubspot.scripts.scraper.main import handler from etl.hubspot.scripts.scraper.main import handler
from tqdm import tqdm from tqdm import tqdm
import json import json
import time
PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
@ -29,13 +28,14 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
deal_ids = hubspot.get_deal_ids_from_company(company_id) deal_ids = hubspot.get_deal_ids_from_company(company_id)
processed = 0 processed = 0
with tqdm(deal_ids, desc=company.name, unit="deal", leave=True, position=0) as deal_bar: with tqdm(
deal_ids, desc=company.name, unit="deal", leave=True, position=0
) as deal_bar:
for deal_id in deal_bar: for deal_id in deal_bar:
deal_data = hubspot.from_deal_id_get_info(deal_id) deal_data = hubspot.from_deal_id_get_info(deal_id)
if deal_data.get("pipeline") != PIPELINE_ID: if deal_data.get("pipeline") != PIPELINE_ID:
deal_bar.set_postfix({"status": "skip", "deal": deal_id}) deal_bar.set_postfix({"status": "skip", "deal": deal_id})
continue continue
time.sleep(5)
deal_bar.set_postfix({"status": "uploading", "deal": deal_id}) deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
handler( handler(
{"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]}, {"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]},