From e1b61bca17d0330535245deead590df4799f43eb Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Apr 2025 14:36:34 +0000 Subject: [PATCH] deem score calculation works for all cavities --- etl/hubSpotClient/hubspot.py | 6 +- etl/hubspot_to_invoice.py | 9 +- etl/surveyPrice/surveyPrice.py | 189 ++++++++++++++++++++++++------ etl/surveyedData/surveryedData.py | 79 ++++++++++++- etl/transform/types.py | 6 + 5 files changed, 244 insertions(+), 45 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index a4a6de5..64aea46 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -53,9 +53,9 @@ class HubSpotClient(): deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], work_type=deal.properties["work_type"], - needs_trickle_ventilation=True if deal.properties["property_needs_trickle_vents"].upper() == "YES" else False, + needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents") else False, post_sap_score=int(deal.properties["domna_survey_post_sap"]), - existing_wall_insulation=deal.properties["existing_wall_insulation"], + existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), installer=deal.properties["installer"], )) @@ -69,4 +69,4 @@ class HubSpotClient(): print(f" - Label: {stage.label}") print(f" ID: {stage.id}") - + diff --git a/etl/hubspot_to_invoice.py b/etl/hubspot_to_invoice.py index 3c3e2fe..ae9d3b3 100644 --- a/etl/hubspot_to_invoice.py +++ b/etl/hubspot_to_invoice.py @@ -11,9 +11,10 @@ from etl.surveyPrice.surveyPrice import SurveyPrice sp = SurveyPrice() -sp.get_cavity_pricing_table("JJC - EMPTIES") -hubspot_df = sp.get_all_surveys_from_hubspot() -sharepoint_df = sp.get_all_surveyed_data_from_sharepoint() +df = sp.calculate_all_price() -df = sp.merge_hub_spot_and_survey_information() \ No newline at end of file +# Surveyoed complete signed off, if normal + +# Make W.C. Folder and upload to commercials in Sharepoint +# Move deal in hub spot \ No newline at end of file diff --git a/etl/surveyPrice/surveyPrice.py b/etl/surveyPrice/surveyPrice.py index 6c46b07..b25ca15 100644 --- a/etl/surveyPrice/surveyPrice.py +++ b/etl/surveyPrice/surveyPrice.py @@ -4,16 +4,19 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor import pandas as pd +# This script will break if we change our current processses, obviously... + class SurveyPrice(): """ A class to work out all prices and uploads to sharepoint for review """ def __init__(self): - self.warmfront_sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) + self.sharepoint_client = SharePointScraper(SharePointInstaller.WARMFRONT) self.master_rate_card_path = None self.all_hubspot_submissions = None self.all_survey_info_from_sharepoint = None self.download_price_card() + self.required_sheets = [ 'JJC - EMPTIES', 'JJC - GENERAL EXTRACTIONS', @@ -27,18 +30,41 @@ class SurveyPrice(): 'SGEC - FORMALDEHYDE EXTRACTION' ] + self.installer = { + "J & J CRUMP": "JJC", + "SCIS": "SCIS", + "SGEC": "SGEC", + "BAXTER KELLY": "BAXTER KELLY", + } + + self.domna_job_to_price_sheet_convertor = { + "JJC - SOLAR": "JJC - SOLAR", + "JJC - EMPTY CAVITY": "JJC - EMPTIES", + "JJC - REMIDIAL FOAM FILLED CAVITY": "JJC - FORMALDEHYDE EXTRACTION", + "JJC - REMIDIAL FILLED CAVITY": "JJC - GENERAL EXTRACTIONS", + "SCIS - SOLAR": "SCIS - SOLAR", + "SCIS - EMPTY CAVITY": "SCIS - EMPTIES", + "SCIS - REMIDIAL FOAM FILLED CAVITY": "SCIS - FORMALDEHYDE EXTRACTION", + "SCIS - REMIDIAL FILLED CAVITY": "SCIS - GENERAL EXTRACTIONS", + "SGEC - EMPTY CAVITY": "SGEC - EMPTIES", + "SGEC - REMIDIAL FOAM FILLED CAVITY": "SGEC - FORMALDEHYDE EXTRACTION", + "SGEC - REMIDIAL FILLED CAVITY": "SGEC - GENERAL EXTRACTIONS", + } + def download_price_card(self): url = None - for files in self.warmfront_sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: + # TODO: Some sanity checks to ensure rate cards title stays consistent + for files in self.sharepoint_client.get_folders_in_path("/Commercials/Rate Cards")['value']: if files['name'] == "MASTER RATE CARD.xlsx": url = files['@microsoft.graph.downloadUrl'] break if url: - content = self.warmfront_sharepoint_client.get_file_content(url) - self.master_rate_card_path = self.warmfront_sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") + content = self.sharepoint_client.get_file_content(url) + self.master_rate_card_path = self.sharepoint_client.create_temp_file(content, "rate_card/rate_card_all.xlsx") return self.master_rate_card_path + def get_cavity_pricing_table(self, sheet_name): excel_file = pd.ExcelFile(self.master_rate_card_path) @@ -58,31 +84,34 @@ class SurveyPrice(): def get_price_matrix(self, sheet_name): df = pd.read_excel(self.master_rate_card_path, sheet_name) - - columns_to_check = { - "no extractors or ventilation required": {"Trickle Vent": 0, "Wetrooms": 0}, - "Trickle Vents ONLY": {"Trickle Vent": 1, "Wetrooms": 0}, - "1 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 1}, - "2 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 2}, - "3 wet room extractor required": {"Trickle Vent": 0, "Wetrooms": 3}, - 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 1}, - 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 2}, - 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "Wetrooms": 3}, - } - pricing_table = [] - for _, row in df.iterrows(): - for key, variables in columns_to_check.items(): - pricing_table.append( - { - "WORK TYPE": row["WORK TYPE"], - "Floor Area Group": row["Total Floor Area"][:-1], - **variables, - "PRICE": row[key] if row[key] != "Not viable" else None, - } - ) - pricing_table = pd.DataFrame(pricing_table) - return pricing_table + if "SOLAR" in sheet_name.upper(): + return df + else: + columns_to_check = { + "no extractors or ventilation required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 0}, + "Trickle Vents ONLY": {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 0}, + "1 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 1}, + "2 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 2}, + "3 wet room extractor required": {"TRICKLE_VENT": 0, "NO_OF_WETROOMS": 3}, + 'Trickle Vents + 1 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 1}, + 'Trickle Vents + 2 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 2}, + 'Trickle Vents + 3 wet room extractor': {"TRICKLE_VENT": 1, "NO_OF_WETROOMS": 3}, + } + + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "WORK TYPE": row["WORK TYPE"], + "FLOOR_AREA_BANDING": row["Total Floor Area"][:-1], + **variables, + "PRICE": row[key] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table def get_all_surveys_from_hubspot(self): hubSpotClient = HubSpotClient() @@ -92,18 +121,21 @@ class SurveyPrice(): for deal in deals: all_deals.append({ + "HUBSPOT_DEAL_ID": deal.deal_id, "HUBSPOT_WORK_TYPE": deal.work_type, "HUBSPOT_DEAL_ADDRESS": deal.deal_name, "HUBSPOT_TRICKLE_VENT":1 if deal.needs_trickle_ventilation else 0, "HUBSPOT_WALL_INSULATION": deal.existing_wall_insulation, "HUBSPOT_POST_INSTALL_SAP_SCORE": deal.post_sap_score, - "HUBSPOT_INSTALLER": deal.installer + "HUBSPOT_INSTALLER": deal.installer, + "HUBSPOT_WETROOMS": deal.no_of_wet_rooms, }) self.all_hubspot_submissions = pd.DataFrame(all_deals) return self.all_hubspot_submissions def get_all_surveyed_data_from_sharepoint(self): + # TODO: A wrapper function self.all_survey_info_from_sharepoint = self.sharepoint_data_for_jjc() return self.all_survey_info_from_sharepoint @@ -118,6 +150,10 @@ class SurveyPrice(): all_survey_info = [] for surveyInfo in jjc_surveys: + cavity_wall_as_built = False + csr = False + foam_insulation = False + info = { "SHAREPOINT INSTALLER": "J & J CRUMP", "SHAREPOINT PRE_SITE_NOTES FOUND": True if surveyInfo.pre_site_note else False, @@ -131,16 +167,47 @@ class SurveyPrice(): if surveyInfo.pre_site_note: floor_banding, total_floor_area = surveyInfo.work_out_total_floor_area() + pre_sap_score = surveyInfo.get_current_sap_score() + pre_sap_score_banding = surveyedDataProcessor.get_band(pre_sap_score) info.update({ "SHAREPOINT TOTAL_FLOOR_AREA": total_floor_area, "SHAREPOINT FLOOR_AREA_BANDING": floor_banding, - "SHAREPOINT PRE_INSTALL_SAP_SCORE": surveyInfo.get_current_sap_score(), + "SHAREPOINT PRE_INSTALL_SAP_SCORE": pre_sap_score, + "SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING": pre_sap_score_banding, }) + + if surveyInfo.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \ + and surveyInfo.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": + cavity_wall_as_built = True if surveyInfo.csr: + csr = True + insulation = surveyInfo.get_insulation_info() info.update({ - "SHAREPOINT INSULATION MATERIAL": surveyInfo.get_insulation_info(), + "SHAREPOINT INSULATION MATERIAL": insulation, }) + if "FOAM" in insulation.upper(): + foam_insulation = True + + if cavity_wall_as_built: + if csr: + if foam_insulation: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FOAM FILLED CAVITY", + }) + else: + info.update({ + "DOMNA JOB TYPE": "REMIDIAL FILLED CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "EMPTY CAVITY" + }) + else: + info.update({ + "DOMNA JOB TYPE": "SOLAR" + }) + all_survey_info.append(info) @@ -163,11 +230,18 @@ class SurveyPrice(): # re-name to installer self.all_survey_info_from_sharepoint = self.all_survey_info_from_sharepoint.rename( - columns={'SHAREPOINT INSTALLER': 'INSTALLER'} + columns={ + 'SHAREPOINT INSTALLER': 'INSTALLER', + 'SHAREPOINT FLOOR_AREA_BANDING': 'FLOOR_AREA_BANDING', + } ) self.all_hubspot_submissions = self.all_hubspot_submissions.rename( - columns={'HUBSPOT_INSTALLER': 'INSTALLER'} + columns={ + 'HUBSPOT_INSTALLER': 'INSTALLER', + 'HUBSPOT_WETROOMS': 'NO_OF_WETROOMS', + 'HUBSPOT_TRICKLE_VENT': 'TRICKLE_VENT', + } ) merged_df = pd.merge( @@ -180,9 +254,47 @@ class SurveyPrice(): merged_df.drop(columns=['clean_address'], inplace=True) - return merged_df + def compute_energy_grant(row): + pre_band_letter = row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"][-1] + post_band_letter = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"])[-1] + return surveyedDataProcessor.gbis_or_eco4_scheme(pre_band_letter, post_band_letter) + + def compute_banding_for_post_sap(row): + post_sap_banding = surveyedDataProcessor.get_band(row["HUBSPOT_POST_INSTALL_SAP_SCORE"]) + return post_sap_banding + + def work_type(row): + if row["ENERGY_GRANT"] == "GBIS": + return row["ENERGY GRANT"] + else: + return f"{row["ENERGY_GRANT"]} - SAP {row["SHAREPOINT PRE_INSTALL_SAP_SCORE_BANDING"]} to {row["POST_INSTALL_SAP_SCORE_BANDING"]}" + + + # Add missing variables + merged_df["ENERGY_GRANT"] = merged_df.apply(compute_energy_grant, axis=1) + merged_df["POST_INSTALL_SAP_SCORE_BANDING"] = merged_df.apply(compute_banding_for_post_sap, axis=1) + merged_df["WORK TYPE"] = merged_df.apply(work_type, axis=1) + + return merged_df + + def calculate_all_price(self): + self.get_all_surveys_from_hubspot() + self.get_all_surveyed_data_from_sharepoint() + submission_data = self.merge_hub_spot_and_survey_information() + final_list = [] + for _, row in submission_data.iterrows(): + if "SOLAR" in row["DOMNA JOB TYPE"].upper(): + raise NotImplementedError("Please implement solar pricing") + else: + # Cavity wall + sheet_name = f"{self.domna_job_to_price_sheet_convertor[f'{self.installer[row["INSTALLER"]]} - {row["DOMNA JOB TYPE"]}'].upper()}" + price_matrix = self.get_price_matrix(sheet_name) + merged_row = pd.merge(row.to_frame().T, price_matrix, on=['WORK TYPE', 'TRICKLE_VENT', 'FLOOR_AREA_BANDING', 'NO_OF_WETROOMS'], how='left') + final_list.append(merged_row) + + return pd.concat(final_list, ignore_index=True) + - @@ -191,4 +303,9 @@ class SurveyPrice(): # Get it working for JJC first, with an idea to make it more diverse # Add some TDD to ensure JJC values are correct # The script can run weekly, for development I can just get one data -# Upload both the W.C. calculations and Rate Card to sharepoint \ No newline at end of file +# Upload both the W.C. calculations and Rate Card to sharepoint +# Due considerations -> Piece of UI, as osmosis, dashboard upload and pushes site notes database and condition report to database. Link the two, make a due consideration report, due consideration report, propery_id () +# Property Tbale + +# Add hubspot deal id into pandas +# Deem score and move deal id to 'Surveyed Completed Signed off" diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index d83f186..1b5248e 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -28,6 +28,80 @@ class surveyedDataProcessor(): insultation = self.csr.insulation_info.type.upper() return insultation return None + + @staticmethod + def get_band(sap_score_number): + bands = [ + ("HIGH A", 96, float("inf")), + ("LOW A", 92, 96), + ("HIGH B", 86, 92), + ("LOW B", 81, 86), + ("HIGH C", 74.5, 81), + ("LOW C", 69, 74.5), + ("HIGH D", 61.5, 69), + ("LOW D", 55, 61.5), + ("HIGH E", 46.5, 55), + ("LOW E", 39, 46.5), + ("HIGH F", 29.5, 39), + ("LOW F", 21, 29.5), + ("HIGH G", 10.5, 21), + ("LOW G", 1, 10.5), + ] + + for band, lower, upper in bands: + if lower <= sap_score_number < upper: + return band + + return None + + @staticmethod + def gbis_or_eco4_scheme(presap_letter, postsap_letter): + """ + *ECO4 (minimum movement)* + D to C + E to C + F to D + G to D + + G -> ABCD + F -> ABCD + E -> ABC + D -> ABC + + *GBIS - Cavity Wall Insulation ONLY* + + D-D + E-D + E-E + F-E + F-F + G-E + G-F + G-G + """ + eco4 = { + "G": ['A', 'B', 'C', 'D'], + "F": ['A', 'B', 'C', 'D'], + "E": ['A', 'B', 'C'], + "D": ['A', 'B', 'C'], + } + + gbis ={ + 'D': ['D'], + 'E': ['E', 'D'], + 'F': ['E', 'F'], + 'G': ['E', 'F', 'G'], + } + + if presap_letter.upper() in eco4: + if postsap_letter.upper() in eco4[presap_letter.upper()]: + return "ECO4" + if presap_letter.upper() in gbis: + if postsap_letter.upper() in gbis[presap_letter.upper()]: + return "GBIS" + + return None + def work_out_total_floor_area(self): @@ -66,7 +140,8 @@ class surveyedDataProcessor(): raise NotImplementedError(f"unknown floor area {floor_area} {self.pre_site_note.summary_information.address}") def get_current_sap_score(self): - return self.pre_site_note.survey_information.current_sap.split(" ")[1] - + score_list = self.pre_site_note.survey_information.current_sap.split(" ") + score = int(score_list[1]) + return score diff --git a/etl/transform/types.py b/etl/transform/types.py index 3342bee..7e25682 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -203,5 +203,11 @@ class PropertyDescription(BaseModel): mainHeating2: Optional[Heating] secondaryHeatingType: Optional[HeatingType] +# class PropertyReport(): + # TODO: Property description + # TODO: Due consideration foregin key + # TODO: Which company did it (Osmosis, Warmfront etc) + # TODO: Links to more foreign keys per report etc + class Insulation(BaseModel): type: str \ No newline at end of file