mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-07-12 13:29:08 +00:00
Merge pull request #47 from Hestia-Homes/feature/hubspot_to_compelte_deem
Feature/hubspot to compelte deem
This commit is contained in:
Jun-te Kim
GitHub
commit
80481c1275
5 changed files with 194 additions and 102 deletions
|
|
@ -1,116 +1,167 @@
|
|||
import os
|
||||
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
|
||||
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
|
||||
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
|
||||
from datetime import timedelta, timezone
|
||||
import datetime
|
||||
import pandas as pd
|
||||
from bs4 import BeautifulSoup
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font
|
||||
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
|
||||
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
|
||||
from collections import defaultdict
|
||||
import time
|
||||
# Auth credentials
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
|
||||
|
||||
hubspot = HubSpotClient()
|
||||
import time
|
||||
pipelines_to_include =[
|
||||
|
||||
# Week calculation
|
||||
today = datetime.datetime.now(datetime.UTC)
|
||||
week1_start = today - timedelta(days=7)
|
||||
week2_start = today - timedelta(days=14)
|
||||
week3_start = today - timedelta(days=21)
|
||||
|
||||
def get_week_label(created_at_str):
|
||||
created_at = datetime.datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S')
|
||||
created_at = created_at.replace(tzinfo=timezone.utc)
|
||||
if week1_start <= created_at <= today:
|
||||
return "Week 1"
|
||||
elif week2_start <= created_at < week1_start:
|
||||
return "Week 2"
|
||||
elif week3_start <= created_at < week2_start:
|
||||
return "Week 3"
|
||||
return None # Ignore notes outside the 3-week range
|
||||
|
||||
# Pipelines to include
|
||||
pipelines_to_include = [
|
||||
"SALES - SOCIAL HOUSING",
|
||||
"PVT PAY",
|
||||
"NRLA GENERAL ENQUIRIES",
|
||||
# "OSMOSIS - SALES",
|
||||
|
||||
]
|
||||
|
||||
exclude_stage = {
|
||||
"SALES - SOCIAL HOUSING" : [
|
||||
"HA TO REENGAGE",
|
||||
"APPOINTMENT SCHEDULED",
|
||||
"AWAITING ASSET LIST",
|
||||
"ASSET LIST RECEIVED",
|
||||
"ASSET LIST STANDARDISED",
|
||||
"ROUTE MARCH CREATED",
|
||||
"SALES - SOCIAL HOUSING": [
|
||||
"HA TO REENGAGE", "APPOINTMENT SCHEDULED", "AWAITING ASSET LIST",
|
||||
"ASSET LIST RECEIVED", "ASSET LIST STANDARDISED", "ROUTE MARCH CREATED",
|
||||
"HA WEEKLY REPORTING",
|
||||
],
|
||||
"PVT PAY": [
|
||||
"LIVE OPPORTUNITY",
|
||||
"CLOSED LOST",
|
||||
"INVOICED",
|
||||
"COLD - KIT",
|
||||
"CLOSED WON",
|
||||
|
||||
"LIVE OPPORTUNITY", "CLOSED LOST", "INVOICED", "COLD - KIT", "CLOSED WON",
|
||||
],
|
||||
"NRLA GENERAL ENQUIRIES": [
|
||||
"CUSTOMER CONTACTED",
|
||||
"LOST",
|
||||
"COLD",
|
||||
"CUSTOMER CONTACTED", "LOST", "COLD",
|
||||
]
|
||||
}
|
||||
|
||||
include_pipeline_upper = [s.upper().strip() for s in pipelines_to_include]
|
||||
exclude_stage_upper = [s.upper().strip() for s in exclude_stage]
|
||||
notes_data = []
|
||||
notes_data = defaultdict(list)
|
||||
pipelines = hubspot.client.crm.pipelines.pipelines_api.get_all(object_type="deals")
|
||||
|
||||
for pipeline in pipelines.results:
|
||||
pipeline_name = pipeline.label.upper().strip()
|
||||
if pipeline_name in pipelines_to_include:
|
||||
for stage in pipeline.stages:
|
||||
if stage.label.upper().strip() not in exclude_stage[pipeline_name]:
|
||||
for deal_id in hubspot.get_all_deals_from_stage_id(stage.id):
|
||||
notes = hubspot.get_notes_from_deals_id(deal_id)
|
||||
if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]:
|
||||
for deals in hubspot.get_all_deals_from_stage_id(stage.id):
|
||||
time.sleep(1)
|
||||
deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []}
|
||||
notes = hubspot.get_notes_from_deals_id(deals["deal_id"])
|
||||
|
||||
for note in notes:
|
||||
deal_name = hubspot.get_deal_name_by_id(deal_id)
|
||||
week_label = get_week_label(note["created_at"])
|
||||
if not week_label:
|
||||
continue
|
||||
html_body = note['note']
|
||||
soup = BeautifulSoup(html_body, "html.parser")
|
||||
plain_text = soup.get_text(separator="\n") # Keeps line breaks
|
||||
notes_data.append({
|
||||
"note_body": plain_text,
|
||||
"deal_name": deal_name, # Include deal_id to relate the note to the deal
|
||||
"pipeline_name": pipeline.label # Add the pipeline name
|
||||
plain_text = soup.get_text(separator="\n")
|
||||
deal_notes_by_week[week_label].append(plain_text)
|
||||
|
||||
if any(deal_notes_by_week.values()):
|
||||
deal_name = hubspot.get_deal_name_by_id(deals["deal_id"])
|
||||
owner_name = "not assigned"
|
||||
if deals["deal_owner"]:
|
||||
try:
|
||||
owner_name = hubspot.get_owner_name_from_id(deals['deal_owner'])
|
||||
except:
|
||||
owner_name = "Couldn't find owner information"
|
||||
|
||||
portal_id = 145275138
|
||||
|
||||
notes_data[pipeline_name].append({
|
||||
"Deal Name": deal_name.upper(),
|
||||
"Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals["deal_id"]}/",
|
||||
"Deal Owner": owner_name,
|
||||
"Deal Stage": stage.label.upper(),
|
||||
"Value": deals["value"],
|
||||
"Notes Week 1": "\n---\n".join(deal_notes_by_week["Week 1"]),
|
||||
"Notes Week 2": "\n---\n".join(deal_notes_by_week["Week 2"]),
|
||||
"Notes Week 3": "\n---\n".join(deal_notes_by_week["Week 3"]),
|
||||
})
|
||||
|
||||
time.sleep(2)
|
||||
print("delay to not bombard the server")
|
||||
time.sleep(1)
|
||||
|
||||
notes_df = pd.DataFrame(notes_data)
|
||||
notes_df.to_csv("output.csv")
|
||||
df = notes_df
|
||||
|
||||
# Create Excel Workbook
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active) # Remove default sheet
|
||||
wb.remove(wb.active)
|
||||
|
||||
for pipeline, group_df in df.groupby("pipeline_name"):
|
||||
ws = wb.create_sheet(title=pipeline[:31]) # Excel sheet name limit = 31 chars
|
||||
for pipeline, deals in notes_data.items():
|
||||
ws = wb.create_sheet(title=pipeline[:31])
|
||||
|
||||
# Sort by deal name
|
||||
group_df = group_df.sort_values("deal_name")
|
||||
headers = ["Deal Name", "Deal URL", "Deal Owner", "Deal Stage", "Value", "Notes Week 1", "Notes Week 2", "Notes Week 3"]
|
||||
ws.append(headers)
|
||||
for cell in ws[1]:
|
||||
cell.font = Font(bold=True)
|
||||
|
||||
current_row = 1
|
||||
for deal_name, deal_notes in group_df.groupby("deal_name"):
|
||||
# Bold header for each deal
|
||||
ws.cell(row=current_row, column=1, value=f"Deal Stage: {deal_name}")
|
||||
ws.cell(row=current_row, column=1).font = Font(bold=True)
|
||||
current_row += 1
|
||||
for row in deals:
|
||||
# Normalize notes to always be lists
|
||||
week_notes = {}
|
||||
for week in range(1, 4):
|
||||
key = f"Notes Week {week}"
|
||||
note_data = row.get(key, [])
|
||||
if isinstance(note_data, str):
|
||||
note_data = [note_data]
|
||||
week_notes[week] = note_data
|
||||
|
||||
# Notes for the deal
|
||||
for note in deal_notes["note_body"]:
|
||||
ws.cell(row=current_row, column=2, value=note)
|
||||
current_row += 1
|
||||
# Get first note per week (if any)
|
||||
first_notes = [week_notes[week][0] if len(week_notes[week]) > 0 else "" for week in range(1, 4)]
|
||||
|
||||
# Add a blank row between groups
|
||||
current_row += 1
|
||||
# Add main deal row + first notes
|
||||
ws.append([
|
||||
row["Deal Name"],
|
||||
row["Deal URL"],
|
||||
row["Deal Owner"],
|
||||
row["Deal Stage"],
|
||||
row["Value"],
|
||||
*first_notes
|
||||
])
|
||||
|
||||
# Save to Excel
|
||||
from datetime import datetime, timedelta
|
||||
today = datetime.today()
|
||||
|
||||
# Determine max number of remaining notes
|
||||
max_additional_notes = max(len(week_notes[week]) for week in range(1, 4)) - 1
|
||||
|
||||
# Add remaining notes
|
||||
for i in range(1, max_additional_notes + 1):
|
||||
note_row = ["", "", "", ""] # Empty deal columns
|
||||
for week in range(1, 4):
|
||||
notes = week_notes[week]
|
||||
note = notes[i] if i < len(notes) else ""
|
||||
note_row.append(note)
|
||||
ws.append(note_row)
|
||||
|
||||
|
||||
# Generate file name with next Monday’s date
|
||||
days_ahead = (7 - today.weekday()) % 7
|
||||
days_ahead = 7 if days_ahead == 0 else days_ahead # If today is Monday, get *next* Monday
|
||||
days_ahead = 7 if days_ahead == 0 else days_ahead
|
||||
next_monday = today + timedelta(days=days_ahead)
|
||||
|
||||
formatted = next_monday.strftime("Monday %d-%m-%Y")
|
||||
|
||||
|
||||
file_name = f"DEAL_NOTES_FROM_HUBSPOT {formatted}.xlsx"
|
||||
wb.save(file_name)
|
||||
formatted = next_monday.strftime("%d-%m-%Y Monday")
|
||||
file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx"
|
||||
output_path = os.path.abspath(file_name)
|
||||
wb.save(output_path)
|
||||
|
||||
# Upload to SharePoint
|
||||
sharepoint_client = SharePointScraper(SharePointInstaller.DOMNA)
|
||||
sharepoint_client.upload_file(output_path, f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",file_name)
|
||||
sharepoint_client.upload_file(
|
||||
output_path,
|
||||
f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",
|
||||
file_name
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from enum import Enum
|
|||
from hubspot.crm.deals import PublicObjectSearchRequest
|
||||
from hubspot.crm.deals.models import SimplePublicObjectInput
|
||||
from etl.hubSpotClient.types import SubmissionInfoFromDeal
|
||||
import time
|
||||
|
||||
|
||||
|
||||
|
|
@ -20,10 +21,17 @@ class HubSpotClient():
|
|||
def get_all_deals(self):
|
||||
return self.client.crm.deals.get_all()
|
||||
|
||||
def get_owner_name_from_id(self, owner_id):
|
||||
owner = self.client.crm.owners.owners_api.get_by_id(owner_id)
|
||||
time.sleep(1)
|
||||
first_name = owner.first_name or ""
|
||||
last_name = owner.last_name or ""
|
||||
return f"{first_name} {last_name}".strip()
|
||||
|
||||
def get_deal_name_by_id(self, deal_id):
|
||||
try:
|
||||
deal = self.client.crm.deals.basic_api.get_by_id(deal_id)
|
||||
time.sleep(1)
|
||||
return deal.properties.get("dealname", "No deal name")
|
||||
except Exception as e:
|
||||
return "Unknown Deal" # Fallback if the deal name is not found
|
||||
|
|
@ -48,6 +56,7 @@ class HubSpotClient():
|
|||
)
|
||||
# Call the search API
|
||||
response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request)
|
||||
time.sleep(1)
|
||||
|
||||
# Add the results to the found_notes list
|
||||
found_notes.extend(response.results)
|
||||
|
|
@ -66,6 +75,7 @@ class HubSpotClient():
|
|||
all_notes.append({
|
||||
"note_id": note.id,
|
||||
"note": note_body,
|
||||
"created_at": note.created_at.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
})
|
||||
return all_notes
|
||||
|
||||
|
|
@ -84,11 +94,14 @@ class HubSpotClient():
|
|||
}],
|
||||
properties=[
|
||||
"dealname",
|
||||
"amount",
|
||||
"hubspot_owner_id",
|
||||
],
|
||||
limit=200,
|
||||
after=after,
|
||||
)
|
||||
response = self.client.crm.deals.search_api.do_search(search_request)
|
||||
time.sleep(1)
|
||||
found_deals.extend(response.results)
|
||||
if not response.paging or not response.paging.next:
|
||||
break
|
||||
|
|
@ -96,7 +109,11 @@ class HubSpotClient():
|
|||
|
||||
all_deals = []
|
||||
for deal in found_deals:
|
||||
all_deals.append(deal.id)
|
||||
all_deals.append({
|
||||
"deal_id": deal.id,
|
||||
"value": deal.properties["amount"],
|
||||
"deal_owner": deal.properties.get("hubspot_owner_id"),
|
||||
})
|
||||
return all_deals
|
||||
|
||||
def get_deals_from_deal_stage(self, deal_stage: DealStage):
|
||||
|
|
|
|||
|
|
@ -32,10 +32,20 @@ deal_ids = df["HUBSPOT_DEAL_ID"].tolist()
|
|||
|
||||
sp.move_deals_to_completed(deal_ids)
|
||||
|
||||
"""
|
||||
TODO:
|
||||
P3) Improve dimirtra script by adding dates to the mixA
|
||||
# Add dates
|
||||
# Add owner's name if possible ( might need to do something to add info in hubspot mnaually)
|
||||
# Add value information
|
||||
# All notes in a particular order
|
||||
# Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes
|
||||
P3 Check to see if emails has arrived
|
||||
P3) Write documentation for tech demos from Khalims demo
|
||||
|
||||
# TODO:
|
||||
# Write documentation for tech demos from Khalims demo
|
||||
# Look into getting 'solar' pricing working. ACIS and Lewis Billignham for examples
|
||||
# Figure out what to do if I see an address that isn't registered but surveyrod
|
||||
# Review deem score with last weeks deem score values to ensure accuracy
|
||||
|
||||
Tuesday
|
||||
P0) output the copy of rate card that was used
|
||||
P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total
|
||||
P2) Review deem score with last weeks deem score values to ensure accuracy
|
||||
P3) Figure out what to do if I see an address that isn't registered but surveyrod
|
||||
"""
|
||||
|
|
@ -58,8 +58,10 @@ class SharePointScraper():
|
|||
|
||||
# Delete me for production
|
||||
if development:
|
||||
self.surveyor_names = ['Jun-te Kim (JJ Crump)']
|
||||
self.surveyor_to_dates_folder = {'Jun-te Kim (JJ Crump)': 'W.C. 31.03.2025'}
|
||||
self.surveyor_names = ["Jun-te Kim (SCIS)"]
|
||||
self.surveyor_to_dates_folder = {
|
||||
'Jun-te Kim (SCIS)': 'W.C. 31.03.2025',
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -145,9 +147,6 @@ class SharePointScraper():
|
|||
|
||||
sharepoint_client.upload_file(file_name, get_file_stream(file_path), sharepoint_path)
|
||||
|
||||
|
||||
|
||||
|
||||
@ensure_surveyor_names_loaded
|
||||
def get_surveryor_names(self):
|
||||
return self.surveyor_names
|
||||
|
|
|
|||
|
|
@ -82,8 +82,14 @@ class SurveyPrice():
|
|||
|
||||
def get_price_matrix(self, sheet_name):
|
||||
df = pd.read_excel(self.master_rate_card_path, sheet_name)
|
||||
pricing_table = []
|
||||
|
||||
if "SOLAR" in sheet_name.upper():
|
||||
for _, row in df.iterrows():
|
||||
pricing_table.append({
|
||||
"WORK TYPE": row["WORK TYPE"],
|
||||
"PRICE": row["PRICE"],
|
||||
})
|
||||
return df
|
||||
else:
|
||||
columns_to_check = {
|
||||
|
|
@ -97,7 +103,6 @@ class SurveyPrice():
|
|||
'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3},
|
||||
}
|
||||
|
||||
pricing_table = []
|
||||
for _, row in df.iterrows():
|
||||
for key, variables in columns_to_check.items():
|
||||
pricing_table.append(
|
||||
|
|
@ -140,26 +145,30 @@ class SurveyPrice():
|
|||
def get_all_surveyed_data_from_sharepoint(self):
|
||||
# TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for
|
||||
# DIfferent installers
|
||||
self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc()
|
||||
# jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC)
|
||||
scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION)
|
||||
# self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True)
|
||||
self.all_survey_info_from_sharepoint = scis_pd
|
||||
return self.all_survey_info_from_sharepoint
|
||||
|
||||
|
||||
def sharepoint_data_for_jjc(self):
|
||||
jjc_sp = SharePointScraper(SharePointInstaller.JJC, development=True)
|
||||
file_paths = jjc_sp.download_file_for_each_address()
|
||||
jjc_surveys = []
|
||||
def sharepoint_data_for_installer(self, installer):
|
||||
sp = SharePointScraper(installer, development=True)
|
||||
file_paths = sp.download_file_for_each_address()
|
||||
surveys = []
|
||||
|
||||
for eachAddress in file_paths:
|
||||
for address, files in eachAddress.items():
|
||||
jjc_surveys.append(surveyedDataProcessor(address, files))
|
||||
surveys.append(surveyedDataProcessor(address, files))
|
||||
|
||||
all_survey_info = []
|
||||
for surveyInfo in jjc_surveys:
|
||||
for surveyInfo in surveys:
|
||||
cavity_wall_as_built = False
|
||||
csr = False
|
||||
foam_insulation = False
|
||||
|
||||
info = {
|
||||
"SHAREPOINT INSTALLER": "J & J CRUMP",
|
||||
"SHAREPOINT INSTALLER": installer.name,
|
||||
"SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False,
|
||||
"SHAREPOINT CSR FOUND": True if surveyInfo.csr else False,
|
||||
"SHAREPOINT TOTAL_FLOOR_AREA": "NO PRE SITE NOTES FOUND",
|
||||
|
|
@ -235,14 +244,12 @@ class SurveyPrice():
|
|||
# re-name to installer
|
||||
self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename(
|
||||
columns={
|
||||
'SHAREPOINT INSTALLER': 'INSTALLER',
|
||||
'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING',
|
||||
}
|
||||
)
|
||||
|
||||
self.all_hubspot_submissions = self.all_hubspot_submissions.rename(
|
||||
columns={
|
||||
'HUBSPOT_INSTALLER': 'INSTALLER',
|
||||
'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS',
|
||||
'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT',
|
||||
}
|
||||
|
|
@ -251,8 +258,8 @@ class SurveyPrice():
|
|||
merged_df = pd.merge(
|
||||
self.all_survey_info_from_sharepoint,
|
||||
self.all_hubspot_submissions,
|
||||
left_on=['clean_address', 'INSTALLER'],
|
||||
right_on=['clean_address', 'INSTALLER'],
|
||||
left_on=['clean_address'],
|
||||
right_on=['clean_address'],
|
||||
how='inner'
|
||||
)
|
||||
|
||||
|
|
@ -288,13 +295,21 @@ class SurveyPrice():
|
|||
final_list = []
|
||||
for _, row in submission_data.iterrows():
|
||||
if "SOLAR" in row["DOMNA JOB TYPE"].upper():
|
||||
raise NotImplementedError("Please implement solar pricing")
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
price_matrix = self.get_price_matrix(sheet_name)
|
||||
merged_row = pd.merge(
|
||||
row.to_frame().T,
|
||||
price_matrix,
|
||||
left_on='DOMNA JOB TYPE',
|
||||
right_on='WORK TYPE',
|
||||
how='outer'
|
||||
)
|
||||
else:
|
||||
# Cavity wall
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
price_matrix = self.get_price_matrix(sheet_name)
|
||||
merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left')
|
||||
final_list.append(merged_row)
|
||||
final_list.append(merged_row)
|
||||
|
||||
return pd.concat(final_list, ignore_index=True)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue