From 985942cc0646a4b16aaf04afaa723f302cc4662f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 26 Mar 2025 12:02:18 +0000 Subject: [PATCH] made jjc automation work if I'm giving the survey sheet --- .devcontainer/devcontainer.json | 3 +- etl/jjc_invoice.py | 181 ++++++++++++++++-------- etl/ratecard/jjcRateCardEmptyCavity.csv | 161 --------------------- 3 files changed, 123 insertions(+), 222 deletions(-) delete mode 100644 etl/ratecard/jjcRateCardEmptyCavity.csv diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b264a11..a99fc9c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,7 +12,8 @@ "ms-python.python", "ms-azuretools.vscode-docker", "ms-toolsai.jupyter", - "mechatroner.rainbow-csv" + "mechatroner.rainbow-csv", + "ms-toolsai.datawrangler" ] } } diff --git a/etl/jjc_invoice.py b/etl/jjc_invoice.py index 6178ff8..2db0916 100644 --- a/etl/jjc_invoice.py +++ b/etl/jjc_invoice.py @@ -275,20 +275,20 @@ file_paths = [{'119 CUTNOOK LANE, M44 6LU': ['/tmp/sharepoint/Andy Rumfitt/W.C. def get_band(score): bands = [ - ("HIGH_A", 96, float("inf")), - ("LOW_A", 92, 96), - ("HIGH_B", 86, 92), - ("LOW_B", 81, 86), - ("HIGH_C", 74.5, 81), - ("LOW_C", 69, 74.5), - ("HIGH_D", 61.5, 69), - ("LOW_D", 55, 61.5), - ("HIGH_E", 46.5, 55), - ("LOW_E", 39, 46.5), - ("HIGH_F", 29.5, 39), - ("LOW_F", 21, 29.5), - ("HIGH_G", 10.5, 21), - ("LOW_G", 1, 10.5), + ("HIGH A", 96, float("inf")), + ("LOW A", 92, 96), + ("HIGH B", 86, 92), + ("LOW B", 81, 86), + ("HIGH C", 74.5, 81), + ("LOW C", 69, 74.5), + ("HIGH D", 61.5, 69), + ("LOW D", 55, 61.5), + ("HIGH E", 46.5, 55), + ("LOW E", 39, 46.5), + ("HIGH F", 29.5, 39), + ("LOW F", 21, 29.5), + ("HIGH G", 10.5, 21), + ("LOW G", 1, 10.5), ] for band, lower, upper in bands: @@ -320,7 +320,17 @@ def work_out_total_floor_area(pre_site_note): total += add_all_floors(pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0 - return math.ceil(total) if total%1 >=0.5 else math.floor(total) + floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total) + if 0 <= floor_area <= 72: + return '0-72m', floor_area + elif 73 <= floor_area <= 97: + return '73-97m', floor_area + elif 98 <= floor_area <= 199: + return '98-199m', floor_area + elif floor_area >= 200: + return 'over 200m', floor_area + else: + return 'Invalid floor area', floor_area def fuzzy_match_address(row, target_address, threshold=90): return fuzz.ratio(row, target_address) >= threshold @@ -378,14 +388,22 @@ def type_of_work(presap, postsap): def get_trickle_vent(str): if "vent" in str.lower(): - return True + return 1 else: - return False + return 0 def get_sap_letter(str): for char in str: if char.isalpha(): return char.upper() + +def get_sap_number(str): + num = "" + for char in str: + if char.isnumeric(): + num+= char + + return int(num) def get_no_of_wet_rooms(str): words = ["WC", "BATHROOM", "KITCHEN"] @@ -396,7 +414,6 @@ def get_no_of_wet_rooms(str): return count def get_insulation_type(str): - print(str.lower()) if "foam" in str.lower(): return "FOAM" else: @@ -404,40 +421,44 @@ def get_insulation_type(str): def get_jjc_price_matrix(file="empty.csv"): - rate_card_df = pd.read_csv(os.path.join(os.getcwd(), "ratecard", "jjcRateCards", file)) - rate_card_df = pd.read_csv(os.path.join(os.getcwd(), "ratecard", "jjcRateCards", "empty.csv")) + df = pd.read_csv(os.path.join(os.getcwd(), "ratecard", "jjcRateCards", file)) - price_table = [] - converter = { - + columns_to_check = { + "no extractors or ventilation required": {"Trickle Vent": 0, "wetrooms": 0}, + "Trickle Vents ONLY": {"Trickle Vent": 1, "wetrooms": 0}, + "1 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 1}, + "2 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 2}, + "3 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 3}, + 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "wetrooms": 1}, + 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "wetrooms": 2}, + 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "wetrooms": 3}, } - for key, value in rate_card_df.iterrows() - print(key) - print(value) - break + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "funding": row["Funding"], + "floor_area_group": row["Total Floor Area"][:-1], + **variables, + "price": row[key][1:] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table - -jjc_rate_card_reader() -jjc_rate_card_reader("foam.csv") -jjc_rate_card_reader("general.csv") - def main(): + price_empty = get_jjc_price_matrix() + price_foam = get_jjc_price_matrix("foam.csv") + price_general = get_jjc_price_matrix("general.csv") + csv_file_path = os.path.join(os.getcwd(), "temp", "jjc_submissionData.csv") submissionData = SubmissionReader(csv_file_path) + total_price = [] - data = { - "Address": [], - "Surveyor's Name": [], - "Type of Work": [], - "Pre SAP": [], - "Total Floor Area": [], - "Empty/Cavity": [], - "Trickle Vent": [], - "No of wet rooms": [], - } # jjc = SharePointScraper(SharePointInstaller.JJC) # file_paths = jjc.download_file_for_each_address() @@ -447,7 +468,7 @@ def main(): list_of_surveys.append(surveyedDataProcessor(address, files)) for survey in list_of_surveys: if survey.pre_site_note: - total_floor_area = work_out_total_floor_area(survey.pre_site_note) + floor_banding, total_floor_area = work_out_total_floor_area(survey.pre_site_note) letter, number = survey.pre_site_note.survey_information.current_sap.split(" ") pre_sap_score = number+letter filtered_df = submissionData.df[submissionData.df["PRE INSTALL SAP SCORE"].apply(lambda x: x == pre_sap_score)] @@ -469,32 +490,72 @@ def main(): return num filtered_df = filtered_df[filtered_df["NO"].apply(lambda x: get_num(house_no) == get_num(x))] - if not filtered_df.empty: - data["Address"].append(survey.address) - data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name) - data["Type of Work"].append(type_of_work(letter.upper(), get_sap_letter(filtered_df["POST INSTALL SAP SCORE"].values[0]))) - data["Pre SAP"].append(survey.pre_site_note.survey_information.current_sap) - data["Total Floor Area"].append(total_floor_area) - data["Empty/Cavity"].append("NONE") - data["Trickle Vent"].append(str(get_trickle_vent(filtered_df["Ventilation Requirements"].values[0]))) - data["No of wet rooms"].append(get_no_of_wet_rooms(filtered_df["Ventilation Requirements"].values[0])) + funding_type = type_of_work(letter.upper(), get_sap_letter(filtered_df["POST INSTALL SAP SCORE"].values[0])) + data = { + "Address": survey.address, + "Surveyor's Name": survey.pre_site_note.assessor_information.name, + "floor_area_group" : floor_banding, + "wetrooms" : get_no_of_wet_rooms(filtered_df["Ventilation Requirements"].values[0]), + "Trickle Vent" : int(get_trickle_vent(filtered_df["Ventilation Requirements"].values[0])), + } + + insultation = None + merged_df = pd.DataFrame() if survey.csr: if survey.csr.insulation_info: - data["Empty/Cavity"].pop() - data["Empty/Cavity"].append(get_insulation_type(survey.csr.insulation_info.type)) - - data["Price"].append("Caluclating...") + insultation = get_insulation_type(survey.csr.insulation_info.type) + + if funding_type == "GBIS": + data.update({"insulation": insultation}) - + if insultation is None: + data.update({"funding": funding_type.upper()}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_empty, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "GENERAL" in insultation.upper(): + data.update({"funding": funding_type.upper() + " Remedial"}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_general, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "FOAM" in insultation.upper(): + data.update({"funding": funding_type.upper() + " Remedial"}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_foam, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + else: + raise RuntimeError(f"unknonw insulation in gbis: {insultation}") + elif funding_type == "ECO4": + data.update({"insulation": insultation}) + if insultation is None: + formatted_funding_type = f"{funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_empty, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "GENERAL" in insultation.upper(): + formatted_funding_type = f"REMEDIAL - {funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_general, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "FOAM" in insultation.upper(): + formatted_funding_type = f"REMEDIAL - {funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_foam, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + else: + raise RuntimeError(f"unknonw insultation in eco4: {insultation}") + else: + raise RuntimeError(f"UNKNOWN FUNDING TYPE {funding_type}") + + if not merged_df.empty: + total_price.append(merged_df) + + final_df = pd.concat(total_price, ignore_index=True) - df = pd.DataFrame(data) # Save to an Excel file - df.to_excel("survey_data.xlsx", index=False) - df.to_csv("survery_data.csv", index=False) + final_df.to_excel("survey_data.xlsx", index=False) + final_df.to_csv("survery_data.csv", index=False) print(f"WEEK COMMENCING {WEEK_COMMENCING}") print("Excel file 'survey_data.xlsx' created successfully!") diff --git a/etl/ratecard/jjcRateCardEmptyCavity.csv b/etl/ratecard/jjcRateCardEmptyCavity.csv deleted file mode 100644 index 6e4a565..0000000 --- a/etl/ratecard/jjcRateCardEmptyCavity.csv +++ /dev/null @@ -1,161 +0,0 @@ -funding_type,floor_area,trickle_vent,number_of_wet_rooms,price -HIGH D TO HIGH C (ECO4),0-72m,0,0,-1 -HIGH D TO HIGH C (ECO4),0-72m,0,1,-1 -HIGH D TO HIGH C (ECO4),0-72m,0,2,-1 -HIGH D TO HIGH C (ECO4),0-72m,0,3,-1 -HIGH D TO HIGH C (ECO4),0-72m,1,0,-1 -HIGH D TO HIGH C (ECO4),0-72m,1,1,-1 -HIGH D TO HIGH C (ECO4),0-72m,1,2,-1 -HIGH D TO HIGH C (ECO4),0-72m,1,3,-1 -HIGH D TO HIGH C (ECO4),73-97m,0,0,-1 -HIGH D TO HIGH C (ECO4),73-97m,0,1,-1 -HIGH D TO HIGH C (ECO4),73-97m,0,2,-1 -HIGH D TO HIGH C (ECO4),73-97m,0,3,-1 -HIGH D TO HIGH C (ECO4),73-97m,1,0,-1 -HIGH D TO HIGH C (ECO4),73-97m,1,1,-1 -HIGH D TO HIGH C (ECO4),73-97m,1,2,-1 -HIGH D TO HIGH C (ECO4),73-97m,1,3,-1 -HIGH D TO HIGH C (ECO4),98-199m,0,0,-1 -HIGH D TO HIGH C (ECO4),98-199m,0,1,-1 -HIGH D TO HIGH C (ECO4),98-199m,0,2,-1 -HIGH D TO HIGH C (ECO4),98-199m,0,3,-1 -HIGH D TO HIGH C (ECO4),98-199m,1,0,-1 -HIGH D TO HIGH C (ECO4),98-199m,1,1,-1 -HIGH D TO HIGH C (ECO4),98-199m,1,2,-1 -HIGH D TO HIGH C (ECO4),98-199m,1,3,-1 -HIGH D TO HIGH C (ECO4),over 200m,0,0,-1 -HIGH D TO HIGH C (ECO4),over 200m,0,1,-1 -HIGH D TO HIGH C (ECO4),over 200m,0,2,-1 -HIGH D TO HIGH C (ECO4),over 200m,0,3,-1 -HIGH D TO HIGH C (ECO4),over 200m,1,0,-1 -HIGH D TO HIGH C (ECO4),over 200m,1,1,-1 -HIGH D TO HIGH C (ECO4),over 200m,1,2,-1 -HIGH D TO HIGH C (ECO4),over 200m,1,3,-1 -HIGH D TO LOW C (ECO4),0-72m,0,0,-1 -HIGH D TO LOW C (ECO4),0-72m,0,1,-1 -HIGH D TO LOW C (ECO4),0-72m,0,2,-1 -HIGH D TO LOW C (ECO4),0-72m,0,3,-1 -HIGH D TO LOW C (ECO4),0-72m,1,0,-1 -HIGH D TO LOW C (ECO4),0-72m,1,1,-1 -HIGH D TO LOW C (ECO4),0-72m,1,2,-1 -HIGH D TO LOW C (ECO4),0-72m,1,3,-1 -HIGH D TO LOW C (ECO4),73-97m,0,0,-1 -HIGH D TO LOW C (ECO4),73-97m,0,1,-1 -HIGH D TO LOW C (ECO4),73-97m,0,2,-1 -HIGH D TO LOW C (ECO4),73-97m,0,3,-1 -HIGH D TO LOW C (ECO4),73-97m,1,0,-1 -HIGH D TO LOW C (ECO4),73-97m,1,1,-1 -HIGH D TO LOW C (ECO4),73-97m,1,2,-1 -HIGH D TO LOW C (ECO4),73-97m,1,3,-1 -HIGH D TO LOW C (ECO4),98-199m,0,0,-1 -HIGH D TO LOW C (ECO4),98-199m,0,1,-1 -HIGH D TO LOW C (ECO4),98-199m,0,2,-1 -HIGH D TO LOW C (ECO4),98-199m,0,3,-1 -HIGH D TO LOW C (ECO4),98-199m,1,0,-1 -HIGH D TO LOW C (ECO4),98-199m,1,1,-1 -HIGH D TO LOW C (ECO4),98-199m,1,2,-1 -HIGH D TO LOW C (ECO4),98-199m,1,3,-1 -HIGH D TO LOW C (ECO4),over 200m,0,0,-1 -HIGH D TO LOW C (ECO4),over 200m,0,1,-1 -HIGH D TO LOW C (ECO4),over 200m,0,2,-1 -HIGH D TO LOW C (ECO4),over 200m,0,3,-1 -HIGH D TO LOW C (ECO4),over 200m,1,0,-1 -HIGH D TO LOW C (ECO4),over 200m,1,1,-1 -HIGH D TO LOW C (ECO4),over 200m,1,2,-1 -HIGH D TO LOW C (ECO4),over 200m,1,3,-1 -LOW D TO HIGH C (ECO4),0-72m,0,0,-1 -LOW D TO HIGH C (ECO4),0-72m,0,1,-1 -LOW D TO HIGH C (ECO4),0-72m,0,2,-1 -LOW D TO HIGH C (ECO4),0-72m,0,3,-1 -LOW D TO HIGH C (ECO4),0-72m,1,0,-1 -LOW D TO HIGH C (ECO4),0-72m,1,1,-1 -LOW D TO HIGH C (ECO4),0-72m,1,2,-1 -LOW D TO HIGH C (ECO4),0-72m,1,3,-1 -LOW D TO HIGH C (ECO4),73-97m,0,0,-1 -LOW D TO HIGH C (ECO4),73-97m,0,1,-1 -LOW D TO HIGH C (ECO4),73-97m,0,2,-1 -LOW D TO HIGH C (ECO4),73-97m,0,3,-1 -LOW D TO HIGH C (ECO4),73-97m,1,0,-1 -LOW D TO HIGH C (ECO4),73-97m,1,1,-1 -LOW D TO HIGH C (ECO4),73-97m,1,2,-1 -LOW D TO HIGH C (ECO4),73-97m,1,3,-1 -LOW D TO HIGH C (ECO4),98-199m,0,0,-1 -LOW D TO HIGH C (ECO4),98-199m,0,1,-1 -LOW D TO HIGH C (ECO4),98-199m,0,2,-1 -LOW D TO HIGH C (ECO4),98-199m,0,3,-1 -LOW D TO HIGH C (ECO4),98-199m,1,0,-1 -LOW D TO HIGH C (ECO4),98-199m,1,1,-1 -LOW D TO HIGH C (ECO4),98-199m,1,2,-1 -LOW D TO HIGH C (ECO4),98-199m,1,3,-1 -LOW D TO HIGH C (ECO4),over 200m,0,0,-1 -LOW D TO HIGH C (ECO4),over 200m,0,1,-1 -LOW D TO HIGH C (ECO4),over 200m,0,2,-1 -LOW D TO HIGH C (ECO4),over 200m,0,3,-1 -LOW D TO HIGH C (ECO4),over 200m,1,0,-1 -LOW D TO HIGH C (ECO4),over 200m,1,1,-1 -LOW D TO HIGH C (ECO4),over 200m,1,2,-1 -LOW D TO HIGH C (ECO4),over 200m,1,3,-1 -LOW D TO LOW C (ECO4),0-72m,0,0,-1 -LOW D TO LOW C (ECO4),0-72m,0,1,-1 -LOW D TO LOW C (ECO4),0-72m,0,2,-1 -LOW D TO LOW C (ECO4),0-72m,0,3,-1 -LOW D TO LOW C (ECO4),0-72m,1,0,-1 -LOW D TO LOW C (ECO4),0-72m,1,1,-1 -LOW D TO LOW C (ECO4),0-72m,1,2,-1 -LOW D TO LOW C (ECO4),0-72m,1,3,-1 -LOW D TO LOW C (ECO4),73-97m,0,0,--1 -LOW D TO LOW C (ECO4),73-97m,0,1,-1 -LOW D TO LOW C (ECO4),73-97m,0,2,-1 -LOW D TO LOW C (ECO4),73-97m,0,3,-1 -LOW D TO LOW C (ECO4),73-97m,1,0,-1 -LOW D TO LOW C (ECO4),73-97m,1,1,-1 -LOW D TO LOW C (ECO4),73-97m,1,2,-1 -LOW D TO LOW C (ECO4),73-97m,1,3,-1 -LOW D TO LOW C (ECO4),98-199m,0,0,-1 -LOW D TO LOW C (ECO4),98-199m,0,1,-1 -LOW D TO LOW C (ECO4),98-199m,0,2,-1 -LOW D TO LOW C (ECO4),98-199m,0,3,-1 -LOW D TO LOW C (ECO4),98-199m,1,0,-1 -LOW D TO LOW C (ECO4),98-199m,1,1,-1 -LOW D TO LOW C (ECO4),98-199m,1,2,-1 -LOW D TO LOW C (ECO4),98-199m,1,3,-1 -LOW D TO LOW C (ECO4),over 200m,0,0,-1 -LOW D TO LOW C (ECO4),over 200m,0,1,-1 -LOW D TO LOW C (ECO4),over 200m,0,2,-1 -LOW D TO LOW C (ECO4),over 200m,0,3,-1 -LOW D TO LOW C (ECO4),over 200m,1,0,-1 -LOW D TO LOW C (ECO4),over 200m,1,1,-1 -LOW D TO LOW C (ECO4),over 200m,1,2,-1 -LOW D TO LOW C (ECO4),over 200m,1,3,-1 -GBIS,0-72m,0,0,-1 -GBIS,0-72m,0,1,-1 -GBIS,0-72m,0,2,-1 -GBIS,0-72m,0,3,-1 -GBIS,0-72m,1,0,-1 -GBIS,0-72m,1,1,-1 -GBIS,0-72m,1,2,-1 -GBIS,0-72m,1,3,-1 -GBIS,73-97m,0,0,-1 -GBIS,73-97m,0,1,-1 -GBIS,73-97m,0,2,-1 -GBIS,73-97m,0,3,-1 -GBIS,73-97m,1,0,-1 -GBIS,73-97m,1,1,-1 -GBIS,73-97m,1,2,-1 -GBIS,73-97m,1,3,-1 -GBIS,98-199m,0,0,-1 -GBIS,98-199m,0,1,-1 -GBIS,98-199m,0,2,-1 -GBIS,98-199m,0,3,-1 -GBIS,98-199m,1,0,-1 -GBIS,98-199m,1,1,-1 -GBIS,98-199m,1,2,-1 -GBIS,98-199m,1,3,-1 -GBIS,over 200m,0,0,-1 -GBIS,over 200m,0,1,-1 -GBIS,over 200m,0,2,-1 -GBIS,over 200m,0,3,-1 -GBIS,over 200m,1,0,-1 -GBIS,over 200m,1,1,-1 -GBIS,over 200m,1,2,-1 -GBIS,over 200m,1,3,-1 \ No newline at end of file