diff --git a/etl/dimitra_hubspot_notes_gather.py b/etl/dimitra_hubspot_notes_gather.py index eb5eaeb..3abacfd 100644 --- a/etl/dimitra_hubspot_notes_gather.py +++ b/etl/dimitra_hubspot_notes_gather.py @@ -1,116 +1,167 @@ import os - -os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" -os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" - -from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday -from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from datetime import timedelta, timezone +import datetime import pandas as pd from bs4 import BeautifulSoup from openpyxl import Workbook from openpyxl.styles import Font - +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday +from etl.hubSpotClient.hubspot import HubSpotClient, DealStage +from collections import defaultdict +import time +# Auth credentials +os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" hubspot = HubSpotClient() -import time -pipelines_to_include =[ + +# Week calculation +today = datetime.datetime.now(datetime.UTC) +week1_start = today - timedelta(days=7) +week2_start = today - timedelta(days=14) +week3_start = today - timedelta(days=21) + +def get_week_label(created_at_str): + created_at = datetime.datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S') + created_at = created_at.replace(tzinfo=timezone.utc) + if week1_start <= created_at <= today: + return "Week 1" + elif week2_start <= created_at < week1_start: + return "Week 2" + elif week3_start <= created_at < week2_start: + return "Week 3" + return None # Ignore notes outside the 3-week range + +# Pipelines to include +pipelines_to_include = [ "SALES - SOCIAL HOUSING", "PVT PAY", "NRLA GENERAL ENQUIRIES", - # "OSMOSIS - SALES", - ] + exclude_stage = { - "SALES - SOCIAL HOUSING" : [ - "HA TO REENGAGE", - "APPOINTMENT SCHEDULED", - "AWAITING ASSET LIST", - "ASSET LIST RECEIVED", - "ASSET LIST STANDARDISED", - "ROUTE MARCH CREATED", + "SALES - SOCIAL HOUSING": [ + "HA TO REENGAGE", "APPOINTMENT SCHEDULED", "AWAITING ASSET LIST", + "ASSET LIST RECEIVED", "ASSET LIST STANDARDISED", "ROUTE MARCH CREATED", "HA WEEKLY REPORTING", ], "PVT PAY": [ - "LIVE OPPORTUNITY", - "CLOSED LOST", - "INVOICED", - "COLD - KIT", - "CLOSED WON", - + "LIVE OPPORTUNITY", "CLOSED LOST", "INVOICED", "COLD - KIT", "CLOSED WON", ], "NRLA GENERAL ENQUIRIES": [ - "CUSTOMER CONTACTED", - "LOST", - "COLD", + "CUSTOMER CONTACTED", "LOST", "COLD", ] } -include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include] -exclude_stage_upper = [s.upper().strip() for s in exclude_stage] -notes_data = [] +notes_data = defaultdict(list) pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals") + for pipeline in pipelines.results: pipeline_name = pipeline.label.upper().strip() if pipeline_name in pipelines_to_include: for stage in pipeline.stages: - if stage.label.upper().strip() not in exclude_stage[pipeline_name]: - for deal_id in hubspot.get_all_deals_from_stage_id(stage.id): - notes = hubspot.get_notes_from_deals_id(deal_id) + if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]: + for deals in hubspot.get_all_deals_from_stage_id(stage.id): + time.sleep(1) + deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []} + notes = hubspot.get_notes_from_deals_id(deals["deal_id"]) + for note in notes: - deal_name = hubspot.get_deal_name_by_id(deal_id) + week_label = get_week_label(note["created_at"]) + if not week_label: + continue html_body = note['note'] soup = BeautifulSoup(html_body, "html.parser") - plain_text = soup.get_text(separator="\n") # Keeps line breaks - notes_data.append({ - "note_body": plain_text, - "deal_name": deal_name, # Include deal_id to relate the note to the deal - "pipeline_name": pipeline.label # Add the pipeline name + plain_text = soup.get_text(separator="\n") + deal_notes_by_week[week_label].append(plain_text) + + if any(deal_notes_by_week.values()): + deal_name = hubspot.get_deal_name_by_id(deals["deal_id"]) + owner_name = "not assigned" + if deals["deal_owner"]: + try: + owner_name = hubspot.get_owner_name_from_id(deals['deal_owner']) + except: + owner_name = "Couldn't find owner information" + + portal_id = 145275138 + + notes_data[pipeline_name].append({ + "Deal Name": deal_name.upper(), + "Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals["deal_id"]}/", + "Deal Owner": owner_name, + "Deal Stage": stage.label.upper(), + "Value": deals["value"], + "Notes Week 1": "\n---\n".join(deal_notes_by_week["Week 1"]), + "Notes Week 2": "\n---\n".join(deal_notes_by_week["Week 2"]), + "Notes Week 3": "\n---\n".join(deal_notes_by_week["Week 3"]), }) - - time.sleep(2) print("delay to not bombard the server") + time.sleep(1) -notes_df = pd.DataFrame(notes_data) -notes_df.to_csv("output.csv") -df = notes_df - +# Create Excel Workbook wb = Workbook() -wb.remove(wb.active) # Remove default sheet +wb.remove(wb.active) -for pipeline, group_df in df.groupby("pipeline_name"): - ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars +for pipeline, deals in notes_data.items(): + ws = wb.create_sheet(title=pipeline[:31]) - # Sort by deal name - group_df = group_df.sort_values("deal_name") + headers = ["Deal Name", "Deal URL", "Deal Owner", "Deal Stage", "Value", "Notes Week 1", "Notes Week 2", "Notes Week 3"] + ws.append(headers) + for cell in ws[1]: + cell.font = Font(bold=True) - current_row = 1 - for deal_name, deal_notes in group_df.groupby("deal_name"): - # Bold header for each deal - ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}") - ws.cell(row=current_row, column=1).font = Font(bold=True) - current_row += 1 + for row in deals: + # Normalize notes to always be lists + week_notes = {} + for week in range(1, 4): + key = f"Notes Week {week}" + note_data = row.get(key, []) + if isinstance(note_data, str): + note_data = [note_data] + week_notes[week] = note_data - # Notes for the deal - for note in deal_notes["note_body"]: - ws.cell(row=current_row, column=2, value=note) - current_row += 1 + # Get first note per week (if any) + first_notes = [week_notes[week][0] if len(week_notes[week]) > 0 else "" for week in range(1, 4)] - # Add a blank row between groups - current_row += 1 + # Add main deal row + first notes + ws.append([ + row["Deal Name"], + row["Deal URL"], + row["Deal Owner"], + row["Deal Stage"], + row["Value"], + *first_notes + ]) -# Save to Excel -from datetime import datetime, timedelta -today = datetime.today() + + # Determine max number of remaining notes + max_additional_notes = max(len(week_notes[week]) for week in range(1, 4)) - 1 + + # Add remaining notes + for i in range(1, max_additional_notes + 1): + note_row = ["", "", "", ""] # Empty deal columns + for week in range(1, 4): + notes = week_notes[week] + note = notes[i] if i < len(notes) else "" + note_row.append(note) + ws.append(note_row) + + +# Generate file name with next Monday’s date days_ahead = (7 - today.weekday()) % 7 -days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday +days_ahead = 7 if days_ahead == 0 else days_ahead next_monday = today + timedelta(days=days_ahead) - -formatted = next_monday.strftime("Monday %d-%m-%Y") - - -file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx" -wb.save(file_name) +formatted = next_monday.strftime("%d-%m-%Y Monday") +file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx" output_path = os.path.abspath(file_name) +wb.save(output_path) + +# Upload to SharePoint sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA) -sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name) +sharepoint_client.upload_file( + output_path, + f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}", + file_name +) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 4924d27..97dee61 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -3,6 +3,7 @@ from enum import Enum from hubspot.crm.deals import PublicObjectSearchRequest from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal +import time @@ -20,10 +21,17 @@ class HubSpotClient(): def get_all_deals(self): return self.client.crm.deals.get_all() + def get_owner_name_from_id(self, owner_id): + owner = self.client.crm.owners.owners_api.get_by_id(owner_id) + time.sleep(1) + first_name = owner.first_name or "" + last_name = owner.last_name or "" + return f"{first_name} {last_name}".strip() def get_deal_name_by_id(self, deal_id): try: deal = self.client.crm.deals.basic_api.get_by_id(deal_id) + time.sleep(1) return deal.properties.get("dealname", "No deal name") except Exception as e: return "Unknown Deal" # Fallback if the deal name is not found @@ -48,6 +56,7 @@ class HubSpotClient(): ) # Call the search API response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request) + time.sleep(1) # Add the results to the found_notes list found_notes.extend(response.results) @@ -66,6 +75,7 @@ class HubSpotClient(): all_notes.append({ "note_id": note.id, "note": note_body, + "created_at": note.created_at.strftime("%Y-%m-%d %H:%M:%S"), }) return all_notes @@ -84,11 +94,14 @@ class HubSpotClient(): }], properties=[ "dealname", + "amount", + "hubspot_owner_id", ], limit=200, after=after, ) response = self.client.crm.deals.search_api.do_search(search_request) + time.sleep(1) found_deals.extend(response.results) if not response.paging or not response.paging.next: break @@ -96,7 +109,11 @@ class HubSpotClient(): all_deals = [] for deal in found_deals: - all_deals.append(deal.id) + all_deals.append({ + "deal_id": deal.id, + "value": deal.properties["amount"], + "deal_owner": deal.properties.get("hubspot_owner_id"), + }) return all_deals def get_deals_from_deal_stage(self, deal_stage: DealStage): diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index d8473f7..86fc2fc 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -32,10 +32,20 @@ deal_ids = df["HUBSPOT_DEAL_ID"].tolist() sp.move_deals_to_completed(deal_ids) +""" + TODO: + P3) Improve dimirtra script by adding dates to the mixA + # Add dates + # Add owner's name if possible ( might need to do something to add info in hubspot mnaually) + # Add value information + # All notes in a particular order + # Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes + P3 Check to see if emails has arrived + P3) Write documentation for tech demos from Khalims demo -# TODO: -# Write documentation for tech demos from Khalims demo -# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples -# Figure out what to do if I see an address that isn't registered but surveyrod -# Review deem score with last weeks deem score values to ensure accuracy - + Tuesday + P0) output the copy of rate card that was used + P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total + P2) Review deem score with last weeks deem score values to ensure accuracy + P3) Figure out what to do if I see an address that isn't registered but surveyrod +""" \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 3e65425..9474eff 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -58,8 +58,10 @@ class SharePointScraper(): # Delete me for production if development: - self.surveyor_names = ['Jun-te Kim (JJ Crump)'] - self.surveyor_to_dates_folder = {'Jun-te Kim (JJ Crump)': 'W.C. 31.03.2025'} + self.surveyor_names = ["Jun-te Kim (SCIS)"] + self.surveyor_to_dates_folder = { + 'Jun-te Kim (SCIS)': 'W.C. 31.03.2025', + } @@ -145,9 +147,6 @@ class SharePointScraper(): sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path) - - - @ensure_surveyor_names_loaded def get_surveryor_names(self): return self.surveyor_names diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 34eda07..ded0f61 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -82,8 +82,14 @@ class SurveyPrice(): def get_price_matrix(self, sheet_name): df = pd.read_excel(self.master_rate_card_path, sheet_name) + pricing_table = [] if "SOLAR" in sheet_name.upper(): + for _, row in df.iterrows(): + pricing_table.append({ + "WORK TYPE": row["WORK TYPE"], + "PRICE": row["PRICE"], + }) return df else: columns_to_check = { @@ -97,7 +103,6 @@ class SurveyPrice(): 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, } - pricing_table = [] for _, row in df.iterrows(): for key, variables in columns_to_check.items(): pricing_table.append( @@ -140,26 +145,30 @@ class SurveyPrice(): def get_all_surveyed_data_from_sharepoint(self): # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for # DIfferent installers - self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() + # jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC) + scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION) + # self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True) + self.all_survey_info_from_sharepoint = scis_pd return self.all_survey_info_from_sharepoint + - def sharepoint_data_for_jjc(self): - jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True) - file_paths = jjc_sp.download_file_for_each_address() - jjc_surveys = [] + def sharepoint_data_for_installer(self, installer): + sp = SharePointScraper(installer, development=True) + file_paths = sp.download_file_for_each_address() + surveys = [] for eachAddress in file_paths: for address, files in eachAddress.items(): - jjc_surveys.append(surveyedDataProcessor(address, files)) + surveys.append(surveyedDataProcessor(address, files)) all_survey_info = [] - for surveyInfo in jjc_surveys: + for surveyInfo in surveys: cavity_wall_as_built = False csr = False foam_insulation = False info = { - "SHAREPOINT INSTALLER": "J & J CRUMP", + "SHAREPOINT INSTALLER": installer.name, "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, "SHAREPOINT CSR FOUND": True if surveyInfo.csr else False, "SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND", @@ -235,14 +244,12 @@ class SurveyPrice(): # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( columns={ - 'SHAREPOINT INSTALLER': 'INSTALLER', 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', } ) self.all_hubspot_submissions = self.all_hubspot_submissions.rename( columns={ - 'HUBSPOT_INSTALLER': 'INSTALLER', 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', } @@ -251,8 +258,8 @@ class SurveyPrice(): merged_df = pd.merge( self.all_survey_info_from_sharepoint, self.all_hubspot_submissions, - left_on=['clean_address', 'INSTALLER'], - right_on=['clean_address', 'INSTALLER'], + left_on=['clean_address'], + right_on=['clean_address'], how='inner' ) @@ -288,13 +295,21 @@ class SurveyPrice(): final_list = [] for _, row in submission_data.iterrows(): if "SOLAR" in row["DOMNA JOB TYPE"].upper(): - raise NotImplementedError("Please implement solar pricing") + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge( + row.to_frame().T, + price_matrix, + left_on='DOMNA JOB TYPE', + right_on='WORK TYPE', + how='outer' + ) else: # Cavity wall - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') - final_list.append(merged_row) + final_list.append(merged_row) return pd.concat(final_list, ignore_index=True)