diff --git a/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py b/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py index 6f35fb5..0539f58 100644 --- a/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py +++ b/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py @@ -2,47 +2,72 @@ from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline from tqdm import tqdm from etl.db.hubSpotLoad import HubspotTodb -''' -# TODO: - get one deal from db, from db - for avri only so far - add it to the db - show in frontend -''' - -# get ALL deals hubspot = HubSpotClient() +hubspot.get_deal_stages() +db = HubspotTodb() -# All deals from a pipeline_id via filter -deals = hubspot.get_deal_ids_by_pipeline( - pipeline_id=Pipeline.OPERATIONS_SOCIAL_HOUSING.value, - ) - -# deals from companies we care about -valueable_deals = [ - Companies.ABRI.value, - Companies.LIVEWEST.value, - Companies.SOUTHERN_HOUSING_GROUP.value, +companies = [ + Companies.ABRI, + Companies.LIVEWEST, + Companies.SOUTHERN_HOUSING_GROUP, ] -deals_to_add = [] +# Track all failures and summary data +all_failed_deals = [] +summary_report = {} +for company in companies: + records = db.find_all_deals_with_company_id(company.value) -deal_to_companies = {} -loader = HubspotTodb() -# Get all deals we care about -for i,deal in enumerate(tqdm(deals)): - company = hubspot.from_deal_get_associated_company_id(deal) - if company in valueable_deals: - deals_to_add.append(deal) - deal_to_companies.update({deal: company}) - deal_data = hubspot.from_deal_get_info(deal_id=deal) - listing_data = hubspot.from_deal_get_associated_listing(deal_id=deal) - loader.new_record_to_hubspot_data(deal_data, deal_to_companies[deal], listing_data) - + updated_count = 0 + checked_count = 0 + failed_deals = [] + for deal in tqdm(records, desc=f"Checking HubSpot deals for {company.name}"): + checked_count += 1 + try: + print(f"πŸ” Working on deal {deal}") + was_up_to_date = db.update_deal(deal, hubspot) -#TODO check if database has abri data -# make companies table -# make a scrip that updates table + if not was_up_to_date: + updated_count += 1 + except Exception as e: + failed_info = { + "company": company.name, + "deal_id": deal, + "error": str(e) + } + failed_deals.append(failed_info) + all_failed_deals.append(failed_info) + print(f"❌ Failed to update deal {deal}: {e}") + + # Store company-level summary (don’t print yet) + summary_report[company.name] = { + "checked": checked_count, + "updated": updated_count, + "up_to_date": checked_count - updated_count - len(failed_deals), + "failed": len(failed_deals), + } + +# ---- Final Summary Report ---- +print("\n" + "="*100) +print("πŸ“Š FINAL SUMMARY REPORT") +print("="*100) + +for company_name, stats in summary_report.items(): + print(f"\n🏒 {company_name}") + print(f" - Total deals checked: {stats['checked']}") + print(f" - Updated deals: {stats['updated']}") + print(f" - Up-to-date deals: {stats['up_to_date']}") + print(f" - Failed deals: {stats['failed']}") + +# ---- Global failure details ---- +if all_failed_deals: + print("\n" + "="*100) + print("⚠️ FAILED DEALS DETAILS") + print("="*100) + for f in all_failed_deals: + print(f" - Company: {f['company']:<25} | Deal ID: {f['deal_id']} | Error: {f['error']}") +else: + print("\nπŸŽ‰ No failed deals across any company!") diff --git a/etl/hubSpotClient/scripts/hubspot_company.py b/etl/hubSpotClient/scripts/hubspot_company.py index b8f8342..2e55b76 100644 --- a/etl/hubSpotClient/scripts/hubspot_company.py +++ b/etl/hubSpotClient/scripts/hubspot_company.py @@ -5,7 +5,7 @@ from etl.db.hubSpotLoad import HubspotTodb hubspot = HubSpotClient() # All deals from a pipeline_id via filter -company = hubspot.get_company_information(Companies.ABRI.value) +company = hubspot.get_company_information(Companies.SOUTHERN_HOUSING_GROUP.value) loader = HubspotTodb() loader.new_record_company(company) diff --git a/etl/hubSpotClient/scripts/hubspot_update_script.py b/etl/hubSpotClient/scripts/hubspot_update_script.py index 9c52d89..5705c23 100644 --- a/etl/hubSpotClient/scripts/hubspot_update_script.py +++ b/etl/hubSpotClient/scripts/hubspot_update_script.py @@ -12,20 +12,76 @@ companies = [ Companies.SOUTHERN_HOUSING_GROUP, ] +# Global trackers +all_failed_deals = [] +summary_report = {} + +print("\nπŸš€ Starting HubSpot deal consistency check...\n") + for company in companies: + print(f"\n🏒 Processing company: {company.name}") records = db.find_all_deals_with_company_id(company.value) - updated_count = 0 # Counter for deals that needed updating - checked_count = 0 # Optional: total processed counter + updated_count = 0 + checked_count = 0 + failed_deals = [] - for deal in tqdm(records, desc="Checking HubSpot deals"): + for deal in tqdm(records, desc=f"Checking HubSpot deals for {company.name}"): checked_count += 1 - was_up_to_date = db.update_deal(deal, hubspot) + try: + print(f"πŸ” Working on deal {deal}") + was_up_to_date = db.update_deal(deal, hubspot) - # update_deal() returns False when discrepancies are found - if not was_up_to_date: - updated_count += 1 + if not was_up_to_date: + updated_count += 1 + print(f"🧩 Deal {deal} was updated.") + else: + print(f"πŸ“ˆ Deal {deal} already up to date.") + except Exception as e: + failed_info = { + "company": company.name, + "deal_id": deal, + "error": str(e) + } + failed_deals.append(failed_info) + all_failed_deals.append(failed_info) + print(f"❌ Failed to update deal {deal}: {e}") + + # Store per-company summary (don’t print yet) + summary_report[company.name] = { + "checked": checked_count, + "updated": updated_count, + "failed": len(failed_deals), + "up_to_date": checked_count - updated_count - len(failed_deals), + } + + # Company-level quick summary print(f"\nβœ… Finished checking {checked_count} deals for company {company.name}.") - print(f"🧩 {updated_count} deal(s) were updated.") - print(f"πŸ“ˆ {checked_count - updated_count} deal(s) were already up to date.") \ No newline at end of file + print(f" 🧩 {updated_count} deal(s) were updated.") + print(f" πŸ“ˆ {summary_report[company.name]['up_to_date']} deal(s) were already up to date.") + print(f" ⚠️ {len(failed_deals)} deal(s) failed.\n") + +# ---- Final Summary Report ---- +print("\n" + "=" * 100) +print("πŸ“Š FINAL SUMMARY REPORT (ALL COMPANIES)") +print("=" * 100) + +for company_name, stats in summary_report.items(): + print(f"\n🏒 {company_name}") + print(f" - Total deals checked: {stats['checked']}") + print(f" - Updated deals: {stats['updated']}") + print(f" - Up-to-date deals: {stats['up_to_date']}") + print(f" - Failed deals: {stats['failed']}") + +# ---- Global Failed Deals ---- +if all_failed_deals: + print("\n" + "=" * 100) + print("⚠️ FAILED DEALS DETAILS") + print("=" * 100) + for f in all_failed_deals: + print(f" - Company: {f['company']:<25} | Deal ID: {f['deal_id']} | Error: {f['error']}") +else: + print("\nπŸŽ‰ No failed deals across any company!") + +print("\n🏁 HubSpot deal consistency check complete!\n")