diff --git a/.vscode/settings.json b/.vscode/settings.json index e8c08c6..9868a02 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,9 @@ { "jupyter.interactiveWindow.textEditor.executeSelection": true, - "python.REPL.sendToNativeREPL": true + "python.REPL.sendToNativeREPL": true, + "notebook.output.scrolling": true, + "notebook.output.textLineLimit": 0 + // Hot reload setting that needs to be in user settings // "jupyter.runStartupCommands": [ diff --git a/etl/age_band_calculator.py b/etl/age_band_calculator.py new file mode 100644 index 0000000..61eb207 --- /dev/null +++ b/etl/age_band_calculator.py @@ -0,0 +1,74 @@ +import os +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING +import pandas as pd +from etl.surveyedData.surveryedData import surveyedDataProcessor + +import etl.scraper.scraper as scraper_module + +def return_pandas_from_scraping(week_commencing, installer): + scraper_module.WEEK_COMMENCING = week_commencing + sp = SharePointScraper(installer) + file_paths = sp.download_file_for_each_address() + list_of_surveys = [] + list_ = [] + for eachAddress in file_paths: + for address, files in eachAddress.items(): + list_of_surveys.append(surveyedDataProcessor(address, files)) + + for survey in list_of_surveys: + dict_ = {} + if survey.pre_site_note: + dict_.update({"address": survey.address}) + dict_.update({"age_band": survey.pre_site_note.property_description.main_property.age_band}) + list_.append(dict_) + + if list_: + return pd.DataFrame(list_) + else: + return None + +installers = [SharePointInstaller.JJC, SharePointInstaller.SOUTH_COAST_INSULATION] +dates = [ + "W.C. 14.04.2025", + "W.C. 31.03.2025", + "W.C. 24.03.2025", + "W.C. 17.03.2025", + "W.C. 10.03.2025", + "W.C. 03.03.2025", + "W.C. 24.02.2025", +] + +all_dfs = [] + +for installer in installers: + for date in dates: + df = return_pandas_from_scraping(date, installer) + if df is not None: + df["installer"] = installer.name + df["week_commencing"] = date + all_dfs.append(df) + +for df in all_dfs: + print(df) + + +giant_df = pd.concat(all_dfs, ignore_index=True) +giant_df.to_excel("age_band.xlsx", index=False) + +giant_df['week_commencing_cleaned'] = pd.to_datetime( + giant_df['week_commencing'].str.replace("W.C. ", ""), + dayfirst=True +) +pd.set_option('display.max_rows', None) + +grouped = giant_df.groupby(['week_commencing_cleaned', 'age_band']).size().unstack(fill_value=0) +grouped = grouped.sort_index() +print(grouped) + +output_file = "grouped_age_band_by_week.xlsx" +grouped.to_excel(output_file) \ No newline at end of file diff --git a/etl/filechecker.py b/etl/filechecker.py deleted file mode 100644 index ce97157..0000000 --- a/etl/filechecker.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" -os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" -os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" -os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" -from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING -import pandas as pd -import hashlib - -def calculate_sha256(bytes_io): - bytes_io.seek(0) # Make sure we're at the start - data = bytes_io.read() - return hashlib.sha256(data).hexdigest() - -south_coast_scraper = SharePointScraper(SharePointInstaller.JJC) - - -folders = south_coast_scraper.get_folders_in_path('/') - - -list_of_file_names = [] -for folder in folders['value']: - if "Khalim" in folder["name"]: - continue - elif ".Training" in folder["name"]: - continue - if 'file' not in folder: - list_of_file_names.append("/" + folder["name"]) - -list_of_dates = [] -for folder in list_of_file_names: - dates = south_coast_scraper.get_folders_in_path(folder) - for date in dates['value']: - if 'file' not in date: - list_of_dates.append(folder + "/" + date["name"]) - -print(list_of_dates) - -list_of_housing_associations = [] -for folder in list_of_dates: - house_ass = south_coast_scraper.get_folders_in_path(folder) - for house in house_ass['value']: - if 'file' not in house: - list_of_housing_associations.append(folder + "/" + house["name"]) - -list_of_address = [] - -for folder in list_of_housing_associations: - address = south_coast_scraper.get_folders_in_path(folder) - for add in address['value']: - if 'file' not in add: - list_of_address.append(folder + "/" + add['name']) - -list_of_pictures = [] - -for folder in list_of_address: - pictures = south_coast_scraper.get_folders_in_path(folder) - for pic in pictures['value']: - if 'file' not in pic: - list_of_pictures.append(folder + "/" + pic['name']) - -print(list_of_pictures) - -final_list = [] -for files in list_of_pictures: - content = south_coast_scraper.get_folders_in_path(files) - for file in content['value']: - if 'file' in file: - url = file['@microsoft.graph.downloadUrl'] - print(f"Downloading {files}/{file['name']}") - sha256 = calculate_sha256(south_coast_scraper.get_file_content(url)) - final_list.append({ - "Directories": files, - "Photo Name": file['name'], - "sha256": sha256, - }) - -final_df = pd.DataFrame(final_list) - -final_df.to_csv("jjc.csv") \ No newline at end of file diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 97dee61..90c551b 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -152,7 +152,7 @@ class HubSpotClient(): deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], work_type=deal.properties["work_type"], - needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False, + needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False, post_sap_score=int(deal.properties["domna_survey_post_sap"]), existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 86fc2fc..eca4271 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -21,12 +21,13 @@ output_path = os.path.abspath(verbose_file) sp.upload_to_sharepoint(output_path, verbose_file) lewis_view = "FOR_LEWIS.xlsx" -selected_columns = ["INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"] +selected_columns = ["HUBSPOT_INSTALLER", "HUBSPOT_DEAL_ADDRESS", "PRICE"] minimal_df = df[selected_columns] minimal_df.to_excel(lewis_view, index=False) output_path = os.path.abspath(lewis_view) sp.upload_to_sharepoint(output_path, lewis_view) +sp.upload_to_sharepoint(sp.get_master_rate_card_path(), "COPY_OF_RATE_CARD_USED.xlsx") deal_ids = df["HUBSPOT_DEAL_ID"].tolist() @@ -34,18 +35,44 @@ sp.move_deals_to_completed(deal_ids) """ TODO: - P3) Improve dimirtra script by adding dates to the mixA - # Add dates - # Add owner's name if possible ( might need to do something to add info in hubspot mnaually) - # Add value information - # All notes in a particular order - # Once i prove its concept, set up a call with Cyrus and Dimitra for a quick call to ensure they like what they see and quick fixes - P3 Check to see if emails has arrived - P3) Write documentation for tech demos from Khalims demo - Tuesday - P0) output the copy of rate card that was used P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total P2) Review deem score with last weeks deem score values to ensure accuracy + + P3) Figure out what to do if I see an address that isn't registered but surveyrod + P3) Write documentation for tech demos from Khalims demo - Handed off to cyrus +""" + +# Look for +# JJC + +# 3 examples of Solar +# No solar example in april deem scroe +# 3 examples Cavity Wall, FOAM, Empty and General ideally +# (in hubspot )111 Duddell Road General ( fibre) - 500, 2 wet rooms +# Empty +# ( in hubspot ) 29 Lower King ( empty ) - 500 - 400 +# Foam +# ( in hubspot ) 6 STOKESAY STREET (foam) - 400 - 200 + +# SCIS +# 3 examples of Solar +# ( in hubspot ) 12 short hedges - Solar 1608 +# ( in hubspot ) 18 short hedge - Solar 1608 +# ( in hubspot) 6 forety road -Solar 1608 + +# 3 examples Cavity Wall, FOAM, Empty and General ideally +# ( in hubspot ) 319 Muirfield Road, (Empty Cavity) - 1000 +# ( hubspot ) 2 queensway, (Fibre) - 500 +# ( in hubspot )56 Aughton Crescent -(foam) - To be worked out by Lewis but lets use this as an oppurtunity - + +# Compare value with what I should get and in the deem score. Keep tabs below so I can check easily + +# Change w.c. date to a weird one to speed up automation + + +# Observation: +""" +2 queensway is wrong due the fact that csr and empty cavity but deem score says cavity """ \ No newline at end of file diff --git a/etl/imagefilenamechcker.py b/etl/imagefilenamechcker.py new file mode 100644 index 0000000..f95666b --- /dev/null +++ b/etl/imagefilenamechcker.py @@ -0,0 +1,94 @@ +import os +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING +import pandas as pd +import hashlib + +def get_photos_name(installer): + south_coast_scraper = SharePointScraper(installer) + folders = south_coast_scraper.get_folders_in_path('/') + + + list_of_file_names = [] + for folder in folders['value']: + if "Khalim" in folder["name"]: + continue + elif ".Training" in folder["name"]: + continue + if 'file' not in folder: + list_of_file_names.append("/" + folder["name"]) + + list_of_dates = [] + for i, folder in enumerate(list_of_file_names): + print(f"getting dates {i}") + dates = south_coast_scraper.get_folders_in_path(folder) + for date in dates['value']: + if 'file' not in date: + list_of_dates.append(folder + "/" + date["name"]) + + + list_of_housing_associations = [] + for i, folder in enumerate(list_of_dates): + print(f"getting housing assoication {i}") + house_ass = south_coast_scraper.get_folders_in_path(folder) + for house in house_ass['value']: + if 'file' not in house: + list_of_housing_associations.append(folder + "/" + house["name"]) + list_of_address = [] + + for i, folder in enumerate(list_of_housing_associations): + print(f"getting address {i}") + address = south_coast_scraper.get_folders_in_path(folder) + for add in address['value']: + if 'file' not in add: + list_of_address.append(folder + "/" + add['name']) + + list_of_pictures = [] + + for i, folder in enumerate(list_of_address): + print(f"getting pictures {i}") + pictures = south_coast_scraper.get_folders_in_path(folder) + for pic in pictures['value']: + if 'file' not in pic: + list_of_pictures.append(folder + "/" + pic['name']) + + print(list_of_pictures) + + final_list = [] + for i,files in enumerate(list_of_pictures): + print(f"for finali list {i}") + + content = south_coast_scraper.get_folders_in_path(files) + parts = files.split("/") + date = None + for part in parts: + if part.startswith("W.C."): + date = part # Output: W.C. 17.03.2025 + for file in content['value']: + if 'file' in file: + final_list.append({ + "Date": date, + "path": file, + "Photo Name": file['name'], + }) + + final_df = pd.DataFrame(final_list) + return final_df + +jjc_df = get_photos_name(SharePointInstaller.JJC) +scis_df = get_photos_name(SharePointInstaller.SOUTH_COAST_INSULATION) + +all_df = [jjc_df, scis_df] + +final_df = pd.concat(all_df, ignore_index=True) +final_df + +final_df.to_csv("photos_name.csv") + +duplicate_names = final_df[final_df.duplicated('Photo Name', keep=False)] +df = final_df +dupe_names_df = df[df.duplicated('Photo Name', keep=False)].sort_values('Photo Name') diff --git a/etl/jjc_old_lewis_manual_way_.py b/etl/jjc_old_lewis_manual_way_.py index 9c14720..ff12e94 100644 --- a/etl/jjc_old_lewis_manual_way_.py +++ b/etl/jjc_old_lewis_manual_way_.py @@ -63,7 +63,6 @@ def work_out_total_floor_area(pre_site_note): total += add_all_floors(pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0 total += add_all_floors(pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0 - floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total) if 0 <= floor_area <= 72: return '0-72m', floor_area diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 9474eff..64d5750 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -8,13 +8,14 @@ from etl.utils.sharepoint.sharepoint import SharePointClient from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator +from tqdm import tqdm from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - return f"W.C. 31.03.2025" + return f"W.C. 31.09.2000" # return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) @@ -153,7 +154,7 @@ class SharePointScraper(): @ensure_surveyor_names_loaded def get_date_folder_names(self): - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): dates_folders = self.get_folders_in_path(f"/{name}") if 'value' not in dates_folders: raise RuntimeError(f"Failed to get dates folder from {name} in {self.sharepoint_drive.name}") @@ -233,7 +234,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def get_number_of_surverys_completed(self): - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): if name in self.surveyor_to_housing_assosications: for house_ass in self.surveyor_to_housing_assosications[name]: address_folders = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") @@ -272,7 +273,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def download_file_for_each_address(self): paths = [] - for name in self.surveyor_names: + for name in tqdm(self.surveyor_names): if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]: for house_ass in self.surveyor_to_housing_assosications[name]: address_files = self.get_folders_in_path(f"/{name}/{WEEK_COMMENCING}/{house_ass}") diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index ded0f61..a5190a5 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -2,6 +2,7 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous from etl.hubSpotClient.hubspot import HubSpotClient, DealStage from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd +from tqdm import tqdm class SurveyPrice(): @@ -35,18 +36,33 @@ class SurveyPrice(): "BAXTER KELLY": "BAXTER KELLY", } - self.domna_job_to_price_sheet_convertor = { - "JJC - SOLAR": "JJC - SOLAR", - "JJC - EMPTY CAVITY": "JJC - EMPTIES", - "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION", - "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS", - "SCIS - SOLAR": "SCIS - SOLAR", - "SCIS - EMPTY CAVITY": "SCIS - EMPTIES", - "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION", - "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS", - "SGEC - EMPTY CAVITY": "SGEC - EMPTIES", - "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION", - "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS", + self.hubspot_job_to_price_sheet_convertor = { + # JJC + "JJC - ECO4 PV": "JJC - SOLAR", + "JJC - ECO4 CWI EMPTY": "JJC - EMPTIES", + "JJC - GBIS CWI EMPTY": "JJC - EMPTIES", + "JJC - ECO4 CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - ECO4 CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + "JJC - GBIS CWI REMEDIAL - FOAM": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - GBIS CWI REMEDIAL - GENERAL": "JJC - GENERAL EXTRACTIONS", + + # SCIS + "SCIS - ECO4 PV": "SCIS - SOLAR", + "SCIS - ECO4 CWI EMPTY": "SCIS - EMPTIES", + "SCIS - GBIS CWI EMPTY": "SCIS - EMPTIES", + "SCIS - ECO4 CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", + "SCIS - ECO4 CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + "SCIS - GBIS CWI REMEDIAL - FOAM": "SCIS - GENERAL EXTRACTIONS", + "SCIS - GBIS CWI REMEDIAL - GENERAL": "SCIS - GENERAL EXTRACTIONS", + + # SGEC + "SGEC - ECO4 CWI EMPTY": "SGEC - EMPTIES", + "SGEC - GBIS CWI EMPTY": "SGEC - EMPTIES", + + "SGEC - ECO4 CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - ECO4 CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", + "SGEC - GBIS CWI REMEDIAL - FOAM": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - GBIS CWI REMEDIAL - GENERAL": "SGEC - GENERAL EXTRACTIONS", } def download_price_card(self): @@ -143,17 +159,16 @@ class SurveyPrice(): def get_all_surveyed_data_from_sharepoint(self): - # TODO: rewrite the function so I pass in sharepointInstaller instead so I can re use the same function for # DIfferent installers - # jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC) + jjc_pd = self.sharepoint_data_for_installer(SharePointInstaller.JJC) scis_pd = self.sharepoint_data_for_installer(SharePointInstaller.SOUTH_COAST_INSULATION) - # self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True) - self.all_survey_info_from_sharepoint = scis_pd + self.all_survey_info_from_sharepoint = pd.concat([jjc_pd, scis_pd], ignore_index=True) return self.all_survey_info_from_sharepoint def sharepoint_data_for_installer(self, installer): - sp = SharePointScraper(installer, development=True) + + sp = SharePointScraper(installer) file_paths = sp.download_file_for_each_address() surveys = [] @@ -175,7 +190,7 @@ class SurveyPrice(): "SHAREPOINT FLOOR_AREA_BANDING": "NO PRE SITE NOTES FOUND", "SHAREPOINT PRE_INSTALL_SAP_SCORE": "NO PRE SITE NOTES FOUND", "SHAREPOINT INSULATION MATERIAL": None, - "SHAREPOINT ADDRESS": address + "SHAREPOINT ADDRESS": surveyInfo.address } if surveyInfo.pre_site_note: @@ -215,10 +230,10 @@ class SurveyPrice(): else: info.update({ "DOMNA JOB TYPE": "EMPTY CAVITY" - }) + }) else: info.update({ - "DOMNA JOB TYPE": "SOLAR" + "DOMNA JOB TYPE": "ECO4 PV" }) @@ -233,14 +248,24 @@ class SurveyPrice(): raise RuntimeError("No information found from Hubspot") # Standardise address - self.all_survey_info_from_sharepoint['clean_address'] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply( - lambda x: x.lower().replace(',', '').strip() - ) + def extract_start_and_postcode(addr): + if not isinstance(addr, str) or addr.strip() == "": + return "", "" + parts = addr.lower().replace(",", "").strip().split() + start = ' '.join(parts[:2]) # Number + street + postcode = ' '.join(parts[-2:]) # Postcode + return start, postcode - self.all_hubspot_submissions['clean_address'] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply( - lambda x: x.lower().replace(',', '').strip() + # Extract start + postcode from both datasets + self.all_survey_info_from_sharepoint[['address_start', 'postcode']] = self.all_survey_info_from_sharepoint['SHAREPOINT ADDRESS'].apply( + lambda x: pd.Series(extract_start_and_postcode(x)) ) + self.all_hubspot_submissions[['address_start', 'postcode']] = self.all_hubspot_submissions['HUBSPOT_DEAL_ADDRESS'].apply( + lambda x: pd.Series(extract_start_and_postcode(x)) + ) + + # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( columns={ @@ -256,14 +281,16 @@ class SurveyPrice(): ) merged_df = pd.merge( - self.all_survey_info_from_sharepoint, - self.all_hubspot_submissions, - left_on=['clean_address'], - right_on=['clean_address'], + self.all_survey_info_from_sharepoint, + self.all_hubspot_submissions, + on=['address_start', 'postcode'], how='inner' ) - merged_df.drop(columns=['clean_address'], inplace=True) + # if hubspot detects + + merged_df.drop(columns=['address_start', 'postcode'], inplace=True) + def compute_energy_grant(row): pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1] @@ -276,12 +303,14 @@ class SurveyPrice(): def work_type(row): if row["ENERGY_GRANT"] == "GBIS": - return row["ENERGY GRANT"] + return "GBIS" else: return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}" # Add missing variables + if merged_df.size == 0: + raise RuntimeError("no matched addresses with hubspot and sharepoint pre site notes") merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1) merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1) merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1) @@ -294,19 +323,26 @@ class SurveyPrice(): submission_data = self.merge_hub_spot_and_survey_information() final_list = [] for _, row in submission_data.iterrows(): - if "SOLAR" in row["DOMNA JOB TYPE"].upper(): - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + if "PV" in row["HUBSPOT_WORK_TYPE"].upper(): + sheet_name = f"{self.hubspot_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"]}'].upper()}" price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge( row.to_frame().T, price_matrix, - left_on='DOMNA JOB TYPE', + left_on='HUBSPOT_WORK_TYPE', right_on='WORK TYPE', how='outer' ) else: # Cavity wall - sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + sheet_name = f'{self.installer[row["HUBSPOT_INSTALLER"]]} - {row["HUBSPOT_WORK_TYPE"].upper()}' + if row['HUBSPOT_WALL_INSULATION'].upper() == "BEAD/FIBRE/WOOL/OTHER": + sheet_name += " - GENERAL" + elif row['HUBSPOT_WALL_INSULATION'].upper() == "EMPTY": + pass + else: + sheet_name += " - FOAM" + sheet_name = self.hubspot_job_to_price_sheet_convertor[sheet_name] price_matrix = self.get_price_matrix(sheet_name) merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') final_list.append(merged_row) diff --git a/poetry.lock b/poetry.lock index ddd2ad3..0f1cdd6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -286,11 +286,11 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main", "dev"] -markers = "sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""} [[package]] name = "comm" @@ -1872,6 +1872,28 @@ files = [ {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "traitlets" version = "5.14.3" @@ -1960,4 +1982,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "9b3e5a8f963d63fbb5fafd8595901358d10aba9f5261b398b9051504ce9320c2" +content-hash = "b5221708d5a15633f7272103bf12970d3da3b05f5861b3e6f3fdfd2b42d8ddad" diff --git a/pyproject.toml b/pyproject.toml index c26faba..096377a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "hubspot-api-client (>=11.1.0,<12.0.0)", "monday (>=2.0.1,<3.0.0)", "beautifulsoup4 (>=4.13.4,<5.0.0)", + "tqdm (>=4.67.1,<5.0.0)", ] [tool.poetry]