From c1239ae8f34a3a21eb9344b08ff63757b0112785 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 16 Apr 2025 18:24:56 +0000 Subject: [PATCH 1/7] quick change --- etl/dimitra_hubspot_notes_gather.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index eb5eaeb..0260661 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -106,10 +106,10 @@ days_ahead = (7 - today.weekday()) % 7 days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday next_monday = today + timedelta(days=days_ahead) -formatted = next_monday.strftime("Monday %d-%m-%Y") +formatted = next_monday.strftime("%d-%m-%Y Monday") -file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx" +file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx" wb.save(file_name) output_path = os.path.abspath(file_name) sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) From 31f0db94ca0fb8e89c227cc6de36b6811a354978 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 08:04:15 +0000 Subject: [PATCH 2/7] updated todo list --- etl/hubspot_to_invoice.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index d8473f7..a3be877 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -34,8 +34,10 @@ sp.move_deals_to_completed(deal_ids) # TODO: -# Write documentation for tech demos from Khalims demo -# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples +# Call with nybble # Figure out what to do if I see an address that isn't registered but surveyrod # Review deem score with last weeks deem score values to ensure accuracy +# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples +# Write documentation for tech demos from Khalims demo +# Improve dimirtra script by adding dates to the mix From 04a27c43537dc328f4912309a39d3b44df047e4d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 12:19:00 +0000 Subject: [PATCH 3/7] solar done --- etl/hubspot_to_invoice.py | 23 +++++++++++------ etl/scraper/scraper.py | 9 +++---- etl/surveyPrice/surveyPrice.py | 47 ++++++++++++++++++++++------------ 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index a3be877..fbf490f 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -32,12 +32,19 @@ deal_ids = df["HUBSPOT_DEAL_ID"].tolist() sp.move_deals_to_completed(deal_ids) +""" + TODO: + P3) Improve dimirtra script by adding dates to the mixA + # Add dates + # Add owner's name if possible ( might need to do something to add info in hubspot mnaually) + # Add value information + # All notes in a particular order + # Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes + P3 Check to see if emails has arrived + P3) Write documentation for tech demos from Khalims demo -# TODO: -# Call with nybble -# Figure out what to do if I see an address that isn't registered but surveyrod -# Review deem score with last weeks deem score values to ensure accuracy -# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples -# Write documentation for tech demos from Khalims demo -# Improve dimirtra script by adding dates to the mix - + Tuesday + P1) - Get ready for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total + P2) Review deem score with last weeks deem score values to ensure accuracy + P3) Figure out what to do if I see an address that isn't registered but surveyrod +""" \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 3e65425..9474eff 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -58,8 +58,10 @@ class SharePointScraper(): # Delete me for production if development: - self.surveyor_names = ['Jun-te Kim (JJ Crump)'] - self.surveyor_to_dates_folder = {'Jun-te Kim (JJ Crump)': 'W.C. 31.03.2025'} + self.surveyor_names = ["Jun-te Kim (SCIS)"] + self.surveyor_to_dates_folder = { + 'Jun-te Kim (SCIS)': 'W.C. 31.03.2025', + } @@ -145,9 +147,6 @@ class SharePointScraper(): sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) - - - @ensure_surveyor_names_loaded def get_surveryor_names(self): return self.surveyor_names diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 34eda07..ded0f61 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -82,8 +82,14 @@ class SurveyPrice(): def get_price_matrix(self, sheet_name): df = pd.read_excel(self.master_rate_card_path, sheet_name) + pricing_table = [] if "SOLAR" in sheet_name.upper(): + for _, row in df.iterrows(): + pricing_table.append({ + "WORK TYPE": row["WORK TYPE"], + "PRICE": row["PRICE"], + }) return df else: columns_to_check = { @@ -97,7 +103,6 @@ class SurveyPrice(): 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, } - pricing_table = [] for _, row in df.iterrows(): for key, variables in columns_to_check.items(): pricing_table.append( @@ -140,26 +145,30 @@ class SurveyPrice(): def get_all_surveyed_data_from_sharepoint(self): # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for # DIfferent installers - self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() + # jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC) + scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION) + # self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True) + self.all_survey_info_from_sharepoint = scis_pd return self.all_survey_info_from_sharepoint + - def sharepoint_data_for_jjc(self): - jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) - file_paths = jjc_sp.download_file_for_each_address() - jjc_surveys = [] + def sharepoint_data_for_installer(self, installer): + sp = SharePointScraper(installer, development=True) + file_paths = sp.download_file_for_each_address() + surveys = [] for eachAddress in file_paths: for address, files in eachAddress.items(): - jjc_surveys.append(surveyedDataProcessor(address, files)) + surveys.append(surveyedDataProcessor(address, files)) all_survey_info = [] - for surveyInfo in jjc_surveys: + for surveyInfo in surveys: cavity_wall_as_built = False csr = False foam_insulation = False info = { - "SHAREPOINT INSTALLER": "J & J CRUMP", + "SHAREPOINT INSTALLER": installer.name, "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", @@ -235,14 +244,12 @@ class SurveyPrice(): # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( columns={ - 'SHAREPOINT INSTALLER': 'INSTALLER', 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', } ) self.all_hubspot_submissions = self.all_hubspot_submissions.rename( columns={ - 'HUBSPOT_INSTALLER': 'INSTALLER', 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', } @@ -251,8 +258,8 @@ class SurveyPrice(): merged_df = pd.merge( self.all_survey_info_from_sharepoint, self.all_hubspot_submissions, - left_on=['clean_address', 'INSTALLER'], - right_on=['clean_address', 'INSTALLER'], + left_on=['clean_address'], + right_on=['clean_address'], how='inner' ) @@ -288,13 +295,21 @@ class SurveyPrice(): final_list = [] for _, row in submission_data.iterrows(): if "SOLAR" in row["DOMNA JOB TYPE"].upper(): - raise NotImplementedError("Please implement solar pricing") + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge( + row.to_frame().T, + price_matrix, + left_on='DOMNA JOB TYPE', + right_on='WORK TYPE', + how='outer' + ) else: # Cavity wall - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') - final_list.append(merged_row) + final_list.append(merged_row) return pd.concat(final_list, ignore_index=True) From c7dc758d28a77c474a798d77b7323bc82fe1109c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 14:04:19 +0000 Subject: [PATCH 4/7] script works? --- etl/dimitra_hubspot_notes_gather.py | 187 +++++++++++++++++----------- etl/hubSpotClient/hubspot.py | 17 ++- etl/hubspot_to_invoice.py | 2 +- 3 files changed, 132 insertions(+), 74 deletions(-) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 0260661..8e19f54 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -1,116 +1,159 @@ import os - -os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" -os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" - -from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday -from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from datetime import timedelta, timezone +import datetime import pandas as pd from bs4 import BeautifulSoup from openpyxl import Workbook from openpyxl.styles import Font - +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from collections import defaultdict +import time +# Auth credentials +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" hubspot = HubSpotClient() -import time -pipelines_to_include =[ + +# Week calculation +today = datetime.datetime.now(datetime.UTC) +week1_start = today - timedelta(days=7) +week2_start = today - timedelta(days=14) +week3_start = today - timedelta(days=21) + +def get_week_label(created_at_str): + created_at = datetime.datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S') + created_at = created_at.replace(tzinfo=timezone.utc) + if week1_start <= created_at <= today: + return "Week 1" + elif week2_start <= created_at < week1_start: + return "Week 2" + elif week3_start <= created_at < week2_start: + return "Week 3" + return None # Ignore notes outside the 3-week range + +# Pipelines to include +pipelines_to_include = [ "SALES - SOCIAL HOUSING", "PVT PAY", "NRLA GENERAL ENQUIRIES", - # "OSMOSIS - SALES", - ] + exclude_stage = { - "SALES - SOCIAL HOUSING" : [ - "HA TO REENGAGE", - "APPOINTMENT SCHEDULED", - "AWAITING ASSET LIST", - "ASSET LIST RECEIVED", - "ASSET LIST STANDARDISED", - "ROUTE MARCH CREATED", + "SALES - SOCIAL HOUSING": [ + "HA TO REENGAGE", "APPOINTMENT SCHEDULED", "AWAITING ASSET LIST", + "ASSET LIST RECEIVED", "ASSET LIST STANDARDISED", "ROUTE MARCH CREATED", "HA WEEKLY REPORTING", ], "PVT PAY": [ - "LIVE OPPORTUNITY", - "CLOSED LOST", - "INVOICED", - "COLD - KIT", - "CLOSED WON", - + "LIVE OPPORTUNITY", "CLOSED LOST", "INVOICED", "COLD - KIT", "CLOSED WON", ], "NRLA GENERAL ENQUIRIES": [ - "CUSTOMER CONTACTED", - "LOST", - "COLD", + "CUSTOMER CONTACTED", "LOST", "COLD", ] } -include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include] -exclude_stage_upper = [s.upper().strip() for s in exclude_stage] -notes_data = [] +notes_data = defaultdict(list) pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals") + for pipeline in pipelines.results: pipeline_name = pipeline.label.upper().strip() if pipeline_name in pipelines_to_include: for stage in pipeline.stages: - if stage.label.upper().strip() not in exclude_stage[pipeline_name]: - for deal_id in hubspot.get_all_deals_from_stage_id(stage.id): - notes = hubspot.get_notes_from_deals_id(deal_id) + if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]: + for deals in hubspot.get_all_deals_from_stage_id(stage.id): + time.sleep(1) + deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []} + notes = hubspot.get_notes_from_deals_id(deals["deal_id"]) + for note in notes: - deal_name = hubspot.get_deal_name_by_id(deal_id) + week_label = get_week_label(note["created_at"]) + if not week_label: + continue html_body = note['note'] soup = BeautifulSoup(html_body, "html.parser") - plain_text = soup.get_text(separator="\n") # Keeps line breaks - notes_data.append({ - "note_body": plain_text, - "deal_name": deal_name, # Include deal_id to relate the note to the deal - "pipeline_name": pipeline.label # Add the pipeline name + plain_text = soup.get_text(separator="\n") + deal_notes_by_week[week_label].append(plain_text) + + if any(deal_notes_by_week.values()): + deal_name = hubspot.get_deal_name_by_id(deals["deal_id"]) + owner_name = "not assigned" + if deals["deal_owner"]: + owner_name = hubspot.get_owner_name_from_id(deals['deal_owner']) + + notes_data[pipeline_name].append({ + "Deal Name": deal_name.upper(), + "Deal Owner": owner_name, + "Deal Stage": stage.label.upper(), + "Value": deals["value"], + "Notes Week 1": "\n---\n".join(deal_notes_by_week["Week 1"]), + "Notes Week 2": "\n---\n".join(deal_notes_by_week["Week 2"]), + "Notes Week 3": "\n---\n".join(deal_notes_by_week["Week 3"]), }) - - time.sleep(2) print("delay to not bombard the server") + time.sleep(2) -notes_df = pd.DataFrame(notes_data) -notes_df.to_csv("output.csv") -df = notes_df - +# Create Excel Workbook wb = Workbook() -wb.remove(wb.active) # Remove default sheet +wb.remove(wb.active) -for pipeline, group_df in df.groupby("pipeline_name"): - ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars +for pipeline, deals in notes_data.items(): + ws = wb.create_sheet(title=pipeline[:31]) - # Sort by deal name - group_df = group_df.sort_values("deal_name") + headers = ["Deal Name", "Deal Owner", "Deal Stage", "Value", "Notes Week 1", "Notes Week 2", "Notes Week 3"] + ws.append(headers) + for cell in ws[1]: + cell.font = Font(bold=True) - current_row = 1 - for deal_name, deal_notes in group_df.groupby("deal_name"): - # Bold header for each deal - ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}") - ws.cell(row=current_row, column=1).font = Font(bold=True) - current_row += 1 + for row in deals: + # Normalize notes to always be lists + week_notes = {} + for week in range(1, 4): + key = f"Notes Week {week}" + note_data = row.get(key, []) + if isinstance(note_data, str): + note_data = [note_data] + week_notes[week] = note_data - # Notes for the deal - for note in deal_notes["note_body"]: - ws.cell(row=current_row, column=2, value=note) - current_row += 1 + # Get first note per week (if any) + first_notes = [week_notes[week][0] if len(week_notes[week]) > 0 else "" for week in range(1, 4)] - # Add a blank row between groups - current_row += 1 + # Add main deal row + first notes + ws.append([ + row["Deal Name"], + row["Deal Owner"], + row["Deal Stage"], + row["Value"], + *first_notes + ]) -# Save to Excel -from datetime import datetime, timedelta -today = datetime.today() + # Determine max number of remaining notes + max_additional_notes = max(len(week_notes[week]) for week in range(1, 4)) - 1 + + # Add remaining notes + for i in range(1, max_additional_notes + 1): + note_row = ["", "", "", ""] # Empty deal columns + for week in range(1, 4): + notes = week_notes[week] + note = notes[i] if i < len(notes) else "" + note_row.append(note) + ws.append(note_row) + + +# Generate file name with next Monday’s date days_ahead = (7 - today.weekday()) % 7 -days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday +days_ahead = 7 if days_ahead == 0 else days_ahead next_monday = today + timedelta(days=days_ahead) - formatted = next_monday.strftime("%d-%m-%Y Monday") - - file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx" -wb.save(file_name) output_path = os.path.abspath(file_name) +wb.save(output_path) + +# Upload to SharePoint sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) -sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name) +sharepoint_client.upload_file( + output_path, + f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}", + file_name +) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 4924d27..d8b04e9 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -3,6 +3,7 @@ from enum import Enum from hubspot.crm.deals import PublicObjectSearchRequest from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal +import time @@ -20,10 +21,17 @@ class HubSpotClient(): def get_all_deals(self): return self.client.crm.deals.get_all() + def get_owner_name_from_id(self, owner_id): + owner = self.client.crm.owners.owners_api.get_by_id(owner_id) + time.sleep(0.5) + first_name = owner.first_name or "" + last_name = owner.last_name or "" + return f"{first_name} {last_name}".strip() def get_deal_name_by_id(self, deal_id): try: deal = self.client.crm.deals.basic_api.get_by_id(deal_id) + time.sleep(0.5) return deal.properties.get("dealname", "No deal name") except Exception as e: return "Unknown Deal" # Fallback if the deal name is not found @@ -66,6 +74,7 @@ class HubSpotClient(): all_notes.append({ "note_id": note.id, "note": note_body, + "created_at": note.created_at.strftime("%Y-%m-%d %H:%M:%S"), }) return all_notes @@ -84,6 +93,8 @@ class HubSpotClient(): }], properties=[ "dealname", + "amount", + "hubspot_owner_id", ], limit=200, after=after, @@ -96,7 +107,11 @@ class HubSpotClient(): all_deals = [] for deal in found_deals: - all_deals.append(deal.id) + all_deals.append({ + "deal_id": deal.id, + "value": deal.properties["amount"], + "deal_owner": deal.properties.get("hubspot_owner_id"), + }) return all_deals def get_deals_from_deal_stage(self, deal_stage: DealStage): diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index fbf490f..d3f742d 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -44,7 +44,7 @@ sp.move_deals_to_completed(deal_ids) P3) Write documentation for tech demos from Khalims demo Tuesday - P1) - Get ready for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total + P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total P2) Review deem score with last weeks deem score values to ensure accuracy P3) Figure out what to do if I see an address that isn't registered but surveyrod """ \ No newline at end of file From 7a08cf2003ceb04ead866d6c97ccda7d5efe4bf2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 14:16:02 +0000 Subject: [PATCH 5/7] add the url --- etl/dimitra_hubspot_notes_gather.py | 9 +++++++-- etl/hubSpotClient/hubspot.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 8e19f54..1c38646 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -82,8 +82,11 @@ for pipeline in pipelines.results: if deals["deal_owner"]: owner_name = hubspot.get_owner_name_from_id(deals['deal_owner']) + portal_id = 145275138 + notes_data[pipeline_name].append({ "Deal Name": deal_name.upper(), + "Deal URL": f"https://app.hubspot.com/contacts/{portal_id}/records/0-3/{deals["deal_id"]}", "Deal Owner": owner_name, "Deal Stage": stage.label.upper(), "Value": deals["value"], @@ -92,7 +95,7 @@ for pipeline in pipelines.results: "Notes Week 3": "\n---\n".join(deal_notes_by_week["Week 3"]), }) print("delay to not bombard the server") - time.sleep(2) + time.sleep(1) # Create Excel Workbook wb = Workbook() @@ -101,7 +104,7 @@ wb.remove(wb.active) for pipeline, deals in notes_data.items(): ws = wb.create_sheet(title=pipeline[:31]) - headers = ["Deal Name", "Deal Owner", "Deal Stage", "Value", "Notes Week 1", "Notes Week 2", "Notes Week 3"] + headers = ["Deal Name", "Deal URL", "Deal Owner", "Deal Stage", "Value", "Notes Week 1", "Notes Week 2", "Notes Week 3"] ws.append(headers) for cell in ws[1]: cell.font = Font(bold=True) @@ -122,12 +125,14 @@ for pipeline, deals in notes_data.items(): # Add main deal row + first notes ws.append([ row["Deal Name"], + row["Deal URL"], row["Deal Owner"], row["Deal Stage"], row["Value"], *first_notes ]) + # Determine max number of remaining notes max_additional_notes = max(len(week_notes[week]) for week in range(1, 4)) - 1 diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index d8b04e9..97dee61 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -23,7 +23,7 @@ class HubSpotClient(): def get_owner_name_from_id(self, owner_id): owner = self.client.crm.owners.owners_api.get_by_id(owner_id) - time.sleep(0.5) + time.sleep(1) first_name = owner.first_name or "" last_name = owner.last_name or "" return f"{first_name} {last_name}".strip() @@ -31,7 +31,7 @@ class HubSpotClient(): def get_deal_name_by_id(self, deal_id): try: deal = self.client.crm.deals.basic_api.get_by_id(deal_id) - time.sleep(0.5) + time.sleep(1) return deal.properties.get("dealname", "No deal name") except Exception as e: return "Unknown Deal" # Fallback if the deal name is not found @@ -56,6 +56,7 @@ class HubSpotClient(): ) # Call the search API response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request) + time.sleep(1) # Add the results to the found_notes list found_notes.extend(response.results) @@ -100,6 +101,7 @@ class HubSpotClient(): after=after, ) response = self.client.crm.deals.search_api.do_search(search_request) + time.sleep(1) found_deals.extend(response.results) if not response.paging or not response.paging.next: break From 00c8a9496a8d24fdeb8a05ac089edc398ef090d2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 15:02:14 +0000 Subject: [PATCH 6/7] merge the chagnse to main --- etl/dimitra_hubspot_notes_gather.py | 5 ++++- etl/hubspot_to_invoice.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index 1c38646..ea069cf 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -80,7 +80,10 @@ for pipeline in pipelines.results: deal_name = hubspot.get_deal_name_by_id(deals["deal_id"]) owner_name = "not assigned" if deals["deal_owner"]: - owner_name = hubspot.get_owner_name_from_id(deals['deal_owner']) + try: + owner_name = hubspot.get_owner_name_from_id(deals['deal_owner']) + except: + owner_name = "Couldn't find owner information" portal_id = 145275138 diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index d3f742d..86fc2fc 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -44,6 +44,7 @@ sp.move_deals_to_completed(deal_ids) P3) Write documentation for tech demos from Khalims demo Tuesday + P0) output the copy of rate card that was used P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total P2) Review deem score with last weeks deem score values to ensure accuracy P3) Figure out what to do if I see an address that isn't registered but surveyrod From 7c62c5c9718941ca032ebbb5977a50ae24521a37 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 17 Apr 2025 15:41:44 +0000 Subject: [PATCH 7/7] record not records --- etl/dimitra_hubspot_notes_gather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index ea069cf..3abacfd 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -89,7 +89,7 @@ for pipeline in pipelines.results: notes_data[pipeline_name].append({ "Deal Name": deal_name.upper(), - "Deal URL": f"https://app.hubspot.com/contacts/{portal_id}/records/0-3/{deals["deal_id"]}", + "Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals["deal_id"]}/", "Deal Owner": owner_name, "Deal Stage": stage.label.upper(), "Value": deals["value"],