mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
Merge pull request #48 from Hestia-Homes/feature/hubspot_to_compelte_deem
Feature/hubspot to compelte deem
This commit is contained in:
Jun-te Kim
GitHub
commit
24c9416a73
11 changed files with 312 additions and 136 deletions
5
.vscode/settings.json
vendored
5
.vscode/settings.json
vendored
|
|
@ -1,6 +1,9 @@
|
|||
{
|
||||
"jupyter.interactiveWindow.textEditor.executeSelection": true,
|
||||
"python.REPL.sendToNativeREPL": true
|
||||
"python.REPL.sendToNativeREPL": true,
|
||||
"notebook.output.scrolling": true,
|
||||
"notebook.output.textLineLimit": 0
|
||||
|
||||
|
||||
// Hot reload setting that needs to be in user settings
|
||||
// "jupyter.runStartupCommands": [
|
||||
|
|
|
|||
74
etl/age_band_calculator.py
Normal file
74
etl/age_band_calculator.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import os
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
|
||||
import pandas as pd
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
|
||||
import etl.scraper.scraper as scraper_module
|
||||
|
||||
def return_pandas_from_scraping(week_commencing, installer):
|
||||
scraper_module.WEEK_COMMENCING = week_commencing
|
||||
sp = SharePointScraper(installer)
|
||||
file_paths = sp.download_file_for_each_address()
|
||||
list_of_surveys = []
|
||||
list_ = []
|
||||
for eachAddress in file_paths:
|
||||
for address, files in eachAddress.items():
|
||||
list_of_surveys.append(surveyedDataProcessor(address, files))
|
||||
|
||||
for survey in list_of_surveys:
|
||||
dict_ = {}
|
||||
if survey.pre_site_note:
|
||||
dict_.update({"address": survey.address})
|
||||
dict_.update({"age_band": survey.pre_site_note.property_description.main_property.age_band})
|
||||
list_.append(dict_)
|
||||
|
||||
if list_:
|
||||
return pd.DataFrame(list_)
|
||||
else:
|
||||
return None
|
||||
|
||||
installers = [SharePointInstaller.JJC, SharePointInstaller.SOUTH_COAST_INSULATION]
|
||||
dates = [
|
||||
"W.C. 14.04.2025",
|
||||
"W.C. 31.03.2025",
|
||||
"W.C. 24.03.2025",
|
||||
"W.C. 17.03.2025",
|
||||
"W.C. 10.03.2025",
|
||||
"W.C. 03.03.2025",
|
||||
"W.C. 24.02.2025",
|
||||
]
|
||||
|
||||
all_dfs = []
|
||||
|
||||
for installer in installers:
|
||||
for date in dates:
|
||||
df = return_pandas_from_scraping(date, installer)
|
||||
if df is not None:
|
||||
df["installer"] = installer.name
|
||||
df["week_commencing"] = date
|
||||
all_dfs.append(df)
|
||||
|
||||
for df in all_dfs:
|
||||
print(df)
|
||||
|
||||
|
||||
giant_df = pd.concat(all_dfs, ignore_index=True)
|
||||
giant_df.to_excel("age_band.xlsx", index=False)
|
||||
|
||||
giant_df['week_commencing_cleaned'] = pd.to_datetime(
|
||||
giant_df['week_commencing'].str.replace("W.C. ", ""),
|
||||
dayfirst=True
|
||||
)
|
||||
pd.set_option('display.max_rows', None)
|
||||
|
||||
grouped = giant_df.groupby(['week_commencing_cleaned', 'age_band']).size().unstack(fill_value=0)
|
||||
grouped = grouped.sort_index()
|
||||
print(grouped)
|
||||
|
||||
output_file = "grouped_age_band_by_week.xlsx"
|
||||
grouped.to_excel(output_file)
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
import os
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
|
||||
import pandas as pd
|
||||
import hashlib
|
||||
|
||||
def calculate_sha256(bytes_io):
|
||||
bytes_io.seek(0) # Make sure we're at the start
|
||||
data = bytes_io.read()
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
south_coast_scraper = SharePointScraper(SharePointInstaller.JJC)
|
||||
|
||||
|
||||
folders = south_coast_scraper.get_folders_in_path('/')
|
||||
|
||||
|
||||
list_of_file_names = []
|
||||
for folder in folders['value']:
|
||||
if "Khalim" in folder["name"]:
|
||||
continue
|
||||
elif ".Training" in folder["name"]:
|
||||
continue
|
||||
if 'file' not in folder:
|
||||
list_of_file_names.append("/" + folder["name"])
|
||||
|
||||
list_of_dates = []
|
||||
for folder in list_of_file_names:
|
||||
dates = south_coast_scraper.get_folders_in_path(folder)
|
||||
for date in dates['value']:
|
||||
if 'file' not in date:
|
||||
list_of_dates.append(folder + "/" + date["name"])
|
||||
|
||||
print(list_of_dates)
|
||||
|
||||
list_of_housing_associations = []
|
||||
for folder in list_of_dates:
|
||||
house_ass = south_coast_scraper.get_folders_in_path(folder)
|
||||
for house in house_ass['value']:
|
||||
if 'file' not in house:
|
||||
list_of_housing_associations.append(folder + "/" + house["name"])
|
||||
|
||||
list_of_address = []
|
||||
|
||||
for folder in list_of_housing_associations:
|
||||
address = south_coast_scraper.get_folders_in_path(folder)
|
||||
for add in address['value']:
|
||||
if 'file' not in add:
|
||||
list_of_address.append(folder + "/" + add['name'])
|
||||
|
||||
list_of_pictures = []
|
||||
|
||||
for folder in list_of_address:
|
||||
pictures = south_coast_scraper.get_folders_in_path(folder)
|
||||
for pic in pictures['value']:
|
||||
if 'file' not in pic:
|
||||
list_of_pictures.append(folder + "/" + pic['name'])
|
||||
|
||||
print(list_of_pictures)
|
||||
|
||||
final_list = []
|
||||
for files in list_of_pictures:
|
||||
content = south_coast_scraper.get_folders_in_path(files)
|
||||
for file in content['value']:
|
||||
if 'file' in file:
|
||||
url = file['@microsoft.graph.downloadUrl']
|
||||
print(f"Downloading {files}/{file['name']}")
|
||||
sha256 = calculate_sha256(south_coast_scraper.get_file_content(url))
|
||||
final_list.append({
|
||||
"Directories": files,
|
||||
"Photo Name": file['name'],
|
||||
"sha256": sha256,
|
||||
})
|
||||
|
||||
final_df = pd.DataFrame(final_list)
|
||||
|
||||
final_df.to_csv("jjc.csv")
|
||||
|
|
@ -152,7 +152,7 @@ class HubSpotClient():
|
|||
deal_id= deal.properties["hs_object_id"],
|
||||
deal_name=deal.properties["dealname"],
|
||||
work_type=deal.properties["work_type"],
|
||||
needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False,
|
||||
needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False,
|
||||
post_sap_score=int(deal.properties["domna_survey_post_sap"]),
|
||||
existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
|
||||
no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
|
||||
|
|
|
|||
|
|
@ -21,12 +21,13 @@ output_path = os.path.abspath(verbose_file)
|
|||
sp.upload_to_sharepoint(output_path, verbose_file)
|
||||
|
||||
lewis_view = "FOR_LEWIS.xlsx"
|
||||
selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"]
|
||||
selected_columns = ["HUBSPOT_INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"]
|
||||
minimal_df = df[selected_columns]
|
||||
minimal_df.to_excel(lewis_view, index=False)
|
||||
output_path = os.path.abspath(lewis_view)
|
||||
sp.upload_to_sharepoint(output_path, lewis_view)
|
||||
|
||||
sp.upload_to_sharepoint(sp.get_master_rate_card_path(), "COPY_OF_RATE_CARD_USED.xlsx")
|
||||
|
||||
deal_ids = df["HUBSPOT_DEAL_ID"].tolist()
|
||||
|
||||
|
|
@ -34,18 +35,44 @@ sp.move_deals_to_completed(deal_ids)
|
|||
|
||||
"""
|
||||
TODO:
|
||||
P3) Improve dimirtra script by adding dates to the mixA
|
||||
# Add dates
|
||||
# Add owner's name if possible ( might need to do something to add info in hubspot mnaually)
|
||||
# Add value information
|
||||
# All notes in a particular order
|
||||
# Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes
|
||||
P3 Check to see if emails has arrived
|
||||
P3) Write documentation for tech demos from Khalims demo
|
||||
|
||||
Tuesday
|
||||
P0) output the copy of rate card that was used
|
||||
P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total
|
||||
P2) Review deem score with last weeks deem score values to ensure accuracy
|
||||
|
||||
|
||||
P3) Figure out what to do if I see an address that isn't registered but surveyrod
|
||||
P3) Write documentation for tech demos from Khalims demo - Handed off to cyrus
|
||||
"""
|
||||
|
||||
# Look for
|
||||
# JJC
|
||||
|
||||
# 3 examples of Solar
|
||||
# No solar example in april deem scroe
|
||||
# 3 examples Cavity Wall, FOAM, Empty and General ideally
|
||||
# (in hubspot )111 Duddell Road General ( fibre) - 500, 2 wet rooms
|
||||
# Empty
|
||||
# ( in hubspot ) 29 Lower King ( empty ) - 500 - 400
|
||||
# Foam
|
||||
# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200
|
||||
|
||||
# SCIS
|
||||
# 3 examples of Solar
|
||||
# ( in hubspot ) 12 short hedges - Solar 1608
|
||||
# ( in hubspot ) 18 short hedge - Solar 1608
|
||||
# ( in hubspot) 6 forety road -Solar 1608
|
||||
|
||||
# 3 examples Cavity Wall, FOAM, Empty and General ideally
|
||||
# ( in hubspot ) 319 Muirfield Road, (Empty Cavity) - 1000
|
||||
# ( hubspot ) 2 queensway, (Fibre) - 500
|
||||
# ( in hubspot )56 Aughton Crescent -(foam) - To be worked out by Lewis but lets use this as an oppurtunity -
|
||||
|
||||
# Compare value with what I should get and in the deem score. Keep tabs below so I can check easily
|
||||
|
||||
# Change w.c. date to a weird one to speed up automation
|
||||
|
||||
|
||||
# Observation:
|
||||
"""
|
||||
2 queensway is wrong due the fact that csr and empty cavity but deem score says cavity
|
||||
"""
|
||||
94
etl/imagefilenamechcker.py
Normal file
94
etl/imagefilenamechcker.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import os
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
|
||||
import pandas as pd
|
||||
import hashlib
|
||||
|
||||
def get_photos_name(installer):
|
||||
south_coast_scraper = SharePointScraper(installer)
|
||||
folders = south_coast_scraper.get_folders_in_path('/')
|
||||
|
||||
|
||||
list_of_file_names = []
|
||||
for folder in folders['value']:
|
||||
if "Khalim" in folder["name"]:
|
||||
continue
|
||||
elif ".Training" in folder["name"]:
|
||||
continue
|
||||
if 'file' not in folder:
|
||||
list_of_file_names.append("/" + folder["name"])
|
||||
|
||||
list_of_dates = []
|
||||
for i, folder in enumerate(list_of_file_names):
|
||||
print(f"getting dates {i}")
|
||||
dates = south_coast_scraper.get_folders_in_path(folder)
|
||||
for date in dates['value']:
|
||||
if 'file' not in date:
|
||||
list_of_dates.append(folder + "/" + date["name"])
|
||||
|
||||
|
||||
list_of_housing_associations = []
|
||||
for i, folder in enumerate(list_of_dates):
|
||||
print(f"getting housing assoication {i}")
|
||||
house_ass = south_coast_scraper.get_folders_in_path(folder)
|
||||
for house in house_ass['value']:
|
||||
if 'file' not in house:
|
||||
list_of_housing_associations.append(folder + "/" + house["name"])
|
||||
list_of_address = []
|
||||
|
||||
for i, folder in enumerate(list_of_housing_associations):
|
||||
print(f"getting address {i}")
|
||||
address = south_coast_scraper.get_folders_in_path(folder)
|
||||
for add in address['value']:
|
||||
if 'file' not in add:
|
||||
list_of_address.append(folder + "/" + add['name'])
|
||||
|
||||
list_of_pictures = []
|
||||
|
||||
for i, folder in enumerate(list_of_address):
|
||||
print(f"getting pictures {i}")
|
||||
pictures = south_coast_scraper.get_folders_in_path(folder)
|
||||
for pic in pictures['value']:
|
||||
if 'file' not in pic:
|
||||
list_of_pictures.append(folder + "/" + pic['name'])
|
||||
|
||||
print(list_of_pictures)
|
||||
|
||||
final_list = []
|
||||
for i,files in enumerate(list_of_pictures):
|
||||
print(f"for finali list {i}")
|
||||
|
||||
content = south_coast_scraper.get_folders_in_path(files)
|
||||
parts = files.split("/")
|
||||
date = None
|
||||
for part in parts:
|
||||
if part.startswith("W.C."):
|
||||
date = part # Output: W.C. 17.03.2025
|
||||
for file in content['value']:
|
||||
if 'file' in file:
|
||||
final_list.append({
|
||||
"Date": date,
|
||||
"path": file,
|
||||
"Photo Name": file['name'],
|
||||
})
|
||||
|
||||
final_df = pd.DataFrame(final_list)
|
||||
return final_df
|
||||
|
||||
jjc_df = get_photos_name(SharePointInstaller.JJC)
|
||||
scis_df = get_photos_name(SharePointInstaller.SOUTH_COAST_INSULATION)
|
||||
|
||||
all_df = [jjc_df, scis_df]
|
||||
|
||||
final_df = pd.concat(all_df, ignore_index=True)
|
||||
final_df
|
||||
|
||||
final_df.to_csv("photos_name.csv")
|
||||
|
||||
duplicate_names = final_df[final_df.duplicated('Photo Name', keep=False)]
|
||||
df = final_df
|
||||
dupe_names_df = df[df.duplicated('Photo Name', keep=False)].sort_values('Photo Name')
|
||||
|
|
@ -63,7 +63,6 @@ def work_out_total_floor_area(pre_site_note):
|
|||
total += add_all_floors(pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0
|
||||
total += add_all_floors(pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0
|
||||
|
||||
|
||||
floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total)
|
||||
if 0 <= floor_area <= 72:
|
||||
return '0-72m', floor_area
|
||||
|
|
|
|||
|
|
@ -8,13 +8,14 @@ from etl.utils.sharepoint.sharepoint import SharePointClient
|
|||
from functools import wraps
|
||||
import re
|
||||
from etl.validator.validator import DomnaSharePointValidator
|
||||
from tqdm import tqdm
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
def previous_monday():
|
||||
today = datetime.today()
|
||||
last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday
|
||||
return f"W.C. 31.03.2025"
|
||||
return f"W.C. 31.09.2000"
|
||||
# return f"W.C. {last_monday.strftime('%d.%m.%Y')}"
|
||||
|
||||
WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())
|
||||
|
|
@ -153,7 +154,7 @@ class SharePointScraper():
|
|||
|
||||
@ensure_surveyor_names_loaded
|
||||
def get_date_folder_names(self):
|
||||
for name in self.surveyor_names:
|
||||
for name in tqdm(self.surveyor_names):
|
||||
dates_folders = self.get_folders_in_path(f"/{name}")
|
||||
if 'value' not in dates_folders:
|
||||
raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}")
|
||||
|
|
@ -233,7 +234,7 @@ class SharePointScraper():
|
|||
|
||||
@ensure_housing_assosiation_is_loaded
|
||||
def get_number_of_surverys_completed(self):
|
||||
for name in self.surveyor_names:
|
||||
for name in tqdm(self.surveyor_names):
|
||||
if name in self.surveyor_to_housing_assosications:
|
||||
for house_ass in self.surveyor_to_housing_assosications[name]:
|
||||
address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")
|
||||
|
|
@ -272,7 +273,7 @@ class SharePointScraper():
|
|||
@ensure_housing_assosiation_is_loaded
|
||||
def download_file_for_each_address(self):
|
||||
paths = []
|
||||
for name in self.surveyor_names:
|
||||
for name in tqdm(self.surveyor_names):
|
||||
if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]:
|
||||
for house_ass in self.surveyor_to_housing_assosications[name]:
|
||||
address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}")
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous
|
|||
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class SurveyPrice():
|
||||
|
|
@ -35,18 +36,33 @@ class SurveyPrice():
|
|||
"BAXTER KELLY": "BAXTER KELLY",
|
||||
}
|
||||
|
||||
self.domna_job_to_price_sheet_convertor = {
|
||||
"JJC - SOLAR": "JJC - SOLAR",
|
||||
"JJC - EMPTY CAVITY": "JJC - EMPTIES",
|
||||
"JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION",
|
||||
"JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS",
|
||||
"SCIS - SOLAR": "SCIS - SOLAR",
|
||||
"SCIS - EMPTY CAVITY": "SCIS - EMPTIES",
|
||||
"SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION",
|
||||
"SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS",
|
||||
"SGEC - EMPTY CAVITY": "SGEC - EMPTIES",
|
||||
"SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION",
|
||||
"SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS",
|
||||
self.hubspot_job_to_price_sheet_convertor = {
|
||||
# JJC
|
||||
"JJC - ECO4 PV": "JJC - SOLAR",
|
||||
"JJC - ECO4 CWI EMPTY": "JJC - EMPTIES",
|
||||
"JJC - GBIS CWI EMPTY": "JJC - EMPTIES",
|
||||
"JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
|
||||
"JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
|
||||
"JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION",
|
||||
"JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS",
|
||||
|
||||
# SCIS
|
||||
"SCIS - ECO4 PV": "SCIS - SOLAR",
|
||||
"SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES",
|
||||
"SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES",
|
||||
"SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
|
||||
"SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
|
||||
"SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS",
|
||||
"SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS",
|
||||
|
||||
# SGEC
|
||||
"SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES",
|
||||
"SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES",
|
||||
|
||||
"SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
|
||||
"SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
|
||||
"SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION",
|
||||
"SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS",
|
||||
}
|
||||
|
||||
def download_price_card(self):
|
||||
|
|
@ -143,17 +159,16 @@ class SurveyPrice():
|
|||
|
||||
|
||||
def get_all_surveyed_data_from_sharepoint(self):
|
||||
# TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for
|
||||
# DIfferent installers
|
||||
# jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC)
|
||||
jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC)
|
||||
scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION)
|
||||
# self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True)
|
||||
self.all_survey_info_from_sharepoint = scis_pd
|
||||
self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True)
|
||||
return self.all_survey_info_from_sharepoint
|
||||
|
||||
|
||||
def sharepoint_data_for_installer(self, installer):
|
||||
sp = SharePointScraper(installer, development=True)
|
||||
|
||||
sp = SharePointScraper(installer)
|
||||
file_paths = sp.download_file_for_each_address()
|
||||
surveys = []
|
||||
|
||||
|
|
@ -175,7 +190,7 @@ class SurveyPrice():
|
|||
"SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND",
|
||||
"SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND",
|
||||
"SHAREPOINT INSULATION MATERIAL": None,
|
||||
"SHAREPOINT ADDRESS": address
|
||||
"SHAREPOINT ADDRESS": surveyInfo.address
|
||||
}
|
||||
|
||||
if surveyInfo.pre_site_note:
|
||||
|
|
@ -215,10 +230,10 @@ class SurveyPrice():
|
|||
else:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "EMPTY CAVITY"
|
||||
})
|
||||
})
|
||||
else:
|
||||
info.update({
|
||||
"DOMNA JOB TYPE": "SOLAR"
|
||||
"DOMNA JOB TYPE": "ECO4 PV"
|
||||
})
|
||||
|
||||
|
||||
|
|
@ -233,14 +248,24 @@ class SurveyPrice():
|
|||
raise RuntimeError("No information found from Hubspot")
|
||||
|
||||
# Standardise address
|
||||
self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply(
|
||||
lambda x: x.lower().replace(',', '').strip()
|
||||
)
|
||||
def extract_start_and_postcode(addr):
|
||||
if not isinstance(addr, str) or addr.strip() == "":
|
||||
return "", ""
|
||||
parts = addr.lower().replace(",", "").strip().split()
|
||||
start = ' '.join(parts[:2]) # Number + street
|
||||
postcode = ' '.join(parts[-2:]) # Postcode
|
||||
return start, postcode
|
||||
|
||||
self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply(
|
||||
lambda x: x.lower().replace(',', '').strip()
|
||||
# Extract start + postcode from both datasets
|
||||
self.all_survey_info_from_sharepoint[['address_start', 'postcode']] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply(
|
||||
lambda x: pd.Series(extract_start_and_postcode(x))
|
||||
)
|
||||
|
||||
self.all_hubspot_submissions[['address_start', 'postcode']] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply(
|
||||
lambda x: pd.Series(extract_start_and_postcode(x))
|
||||
)
|
||||
|
||||
|
||||
# re-name to installer
|
||||
self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename(
|
||||
columns={
|
||||
|
|
@ -256,14 +281,16 @@ class SurveyPrice():
|
|||
)
|
||||
|
||||
merged_df = pd.merge(
|
||||
self.all_survey_info_from_sharepoint,
|
||||
self.all_hubspot_submissions,
|
||||
left_on=['clean_address'],
|
||||
right_on=['clean_address'],
|
||||
self.all_survey_info_from_sharepoint,
|
||||
self.all_hubspot_submissions,
|
||||
on=['address_start', 'postcode'],
|
||||
how='inner'
|
||||
)
|
||||
|
||||
merged_df.drop(columns=['clean_address'], inplace=True)
|
||||
# if hubspot detects
|
||||
|
||||
merged_df.drop(columns=['address_start', 'postcode'], inplace=True)
|
||||
|
||||
|
||||
def compute_energy_grant(row):
|
||||
pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1]
|
||||
|
|
@ -276,12 +303,14 @@ class SurveyPrice():
|
|||
|
||||
def work_type(row):
|
||||
if row["ENERGY_GRANT"] == "GBIS":
|
||||
return row["ENERGY GRANT"]
|
||||
return "GBIS"
|
||||
else:
|
||||
return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}"
|
||||
|
||||
|
||||
# Add missing variables
|
||||
if merged_df.size == 0:
|
||||
raise RuntimeError("no matched addresses with hubspot and sharepoint pre site notes")
|
||||
merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1)
|
||||
merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1)
|
||||
merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1)
|
||||
|
|
@ -294,19 +323,26 @@ class SurveyPrice():
|
|||
submission_data = self.merge_hub_spot_and_survey_information()
|
||||
final_list = []
|
||||
for _, row in submission_data.iterrows():
|
||||
if "SOLAR" in row["DOMNA JOB TYPE"].upper():
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
if "PV" in row["HUBSPOT_WORK_TYPE"].upper():
|
||||
sheet_name = f"{self.hubspot_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}"
|
||||
price_matrix = self.get_price_matrix(sheet_name)
|
||||
merged_row = pd.merge(
|
||||
row.to_frame().T,
|
||||
price_matrix,
|
||||
left_on='DOMNA JOB TYPE',
|
||||
left_on='HUBSPOT_WORK_TYPE',
|
||||
right_on='WORK TYPE',
|
||||
how='outer'
|
||||
)
|
||||
else:
|
||||
# Cavity wall
|
||||
sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}"
|
||||
sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}'
|
||||
if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER":
|
||||
sheet_name += " - GENERAL"
|
||||
elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY":
|
||||
pass
|
||||
else:
|
||||
sheet_name += " - FOAM"
|
||||
sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name]
|
||||
price_matrix = self.get_price_matrix(sheet_name)
|
||||
merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left')
|
||||
final_list.append(merged_row)
|
||||
|
|
|
|||
26
poetry.lock
generated
26
poetry.lock
generated
|
|
@ -286,11 +286,11 @@ description = "Cross-platform colored terminal text."
|
|||
optional = false
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
|
||||
groups = ["main", "dev"]
|
||||
markers = "sys_platform == \"win32\""
|
||||
files = [
|
||||
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
|
||||
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
|
||||
]
|
||||
markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""}
|
||||
|
||||
[[package]]
|
||||
name = "comm"
|
||||
|
|
@ -1872,6 +1872,28 @@ files = [
|
|||
{file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.67.1"
|
||||
description = "Fast, Extensible Progress Meter"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"},
|
||||
{file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[package.extras]
|
||||
dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"]
|
||||
discord = ["requests"]
|
||||
notebook = ["ipywidgets (>=6)"]
|
||||
slack = ["slack-sdk"]
|
||||
telegram = ["requests"]
|
||||
|
||||
[[package]]
|
||||
name = "traitlets"
|
||||
version = "5.14.3"
|
||||
|
|
@ -1960,4 +1982,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.12"
|
||||
content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2"
|
||||
content-hash = "b5221708d5a15633f7272103bf12970d3da3b05f5861b3e6f3fdfd2b42d8ddad"
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ dependencies = [
|
|||
"hubspot-api-client (>=11.1.0,<12.0.0)",
|
||||
"monday (>=2.0.1,<3.0.0)",
|
||||
"beautifulsoup4 (>=4.13.4,<5.0.0)",
|
||||
"tqdm (>=4.67.1,<5.0.0)",
|
||||
]
|
||||
|
||||
[tool.poetry]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue