junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

get rid of photos from local drive when i run it locally so its less verbose

Browse source
This commit is contained in:
Jun-te Kim 2026-03-31 18:46:37 +00:00
parent ea84cf9fd4
commit fcf25f7cac
2 changed files with 23 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
etl/hubspot/hubspotDataTodB.py
View file

@ -311,6 +311,9 @@ class HubspotDataToDb:
f"⚠️ Failed to download/upload photo for deal_id {deal_in_db.deal_id}: {e}"
)
# Continue without the file — don't crash the entire update
finally:
if "local_file" in locals() and os.path.exists(local_file):
os.remove(local_file)
else:
print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}")
@ -405,6 +408,9 @@ class HubspotDataToDb:
f"⚠️ Failed to download photo for deal_id {existing.deal_id}: {e}"
)
# Continue without the file — don't crash the update
finally:
if "local_file" in locals() and os.path.exists(local_file):
os.remove(local_file)
else:
print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}")
@ -475,6 +481,9 @@ class HubspotDataToDb:
f"⚠️ Failed to download photo for deal_id {new_record.deal_id}: {e}"
)
# Continue without the file — don't crash the insert
finally:
if "local_file" in locals() and os.path.exists(local_file):
os.remove(local_file)
session.add(new_record)
session.commit()

21
etl/hubspot/scripts/scraper/bulk_load.py
View file

@ -16,17 +16,24 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
hubspot = HubspotClient()
targets = companies or list(Companies)
for company in tqdm(targets, desc="Companies"):
for company in tqdm(targets, desc="Companies", unit="co"):
company_id = company.value
deal_ids = hubspot.get_deal_ids_from_company(company_id)
for deal_id in tqdm(deal_ids, desc=f"{company.name}", leave=False):
deal_data = hubspot.from_deal_id_get_info(deal_id)
if deal_data.get("pipeline") != PIPELINE_ID:
continue
processed = 0
with tqdm(deal_ids, desc=company.name, unit="deal", leave=False) as deal_bar:
for deal_id in deal_bar:
deal_data = hubspot.from_deal_id_get_info(deal_id)
if deal_data.get("pipeline") != PIPELINE_ID:
deal_bar.set_postfix({"status": "skip", "deal": deal_id})
continue
handler({"hubspot_deal_id": deal_id}, context=None)
print(f"Processed deal {deal_id} (company: {company.name})")
deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
handler({"hubspot_deal_id": deal_id}, context=None)
processed += 1
deal_bar.set_postfix({"status": "done", "deal": deal_id})
tqdm.write(f"[{company.name}] {processed}/{len(deal_ids)} deals in pipeline")
if __name__ == "__main__":