From 6ab3804b4f05ccf0af0b2bf67e13c6135cdb0e8f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 15 Dec 2023 17:32:16 +0000 Subject: [PATCH] done with ha33 analysis for now --- etl/eligibility/ha_15_32/app.py | 59 ++++++------ etl/eligibility/ha_15_32/ha33_app.py | 135 ++++++++++++++++++++++++++- 2 files changed, 162 insertions(+), 32 deletions(-) diff --git a/etl/eligibility/ha_15_32/app.py b/etl/eligibility/ha_15_32/app.py index 2d41f288..3c7ae901 100644 --- a/etl/eligibility/ha_15_32/app.py +++ b/etl/eligibility/ha_15_32/app.py @@ -376,37 +376,38 @@ def prepare_model_data_row(property_id, modelling_epc, cleaned, cleaning_data, c # after retrofit. We use the minimal u-values required to meet building regulations part L # TODO: Check the performance of the materials warmfront's installers use, particularly for # cavity - simulation_recommendations = [ - { - "recommendation_id": "-".join([property_id, "cavity"]), - "type": "cavity_wall_insulation", - "new_u_value": 0.35, - "parts": [{}] - }, - { - "recommendation_id": "-".join([property_id, "loft"]), - "type": "loft_insulation", - "new_u_value": 0.16, - "parts": [{"depth": 270}] - } - ] - scoring_dict = {} - for recommendation in simulation_recommendations: - scoring_dict = create_recommendation_scoring_data( - property=p, - recommendation=recommendation, - starting_epc_data=starting_epc_data, - ending_epc_data=ending_epc_data, - fixed_data=fixed_data, - ) - # At each iteration, we want to update the ending_epc_data, so in the end, ending_epc_data contains - # all of the updates - for k in scoring_dict.keys(): - if k in ending_epc_data.columns: - ending_epc_data[k] = scoring_dict[k] + cavity_simulation = { + "recommendation_id": "-".join([property_id, "cavity"]), + "type": "cavity_wall_insulation", + "new_u_value": 0.35, + "parts": [{}] + } - return scoring_dict + loft_simulation = { + "recommendation_id": "-".join([property_id, "loft"]), + "type": "loft_insulation", + "new_u_value": 0.16, + "parts": [{"depth": 270}] + } + + cavity_scoring = create_recommendation_scoring_data( + property=p, + recommendation=cavity_simulation, + starting_epc_data=starting_epc_data, + ending_epc_data=ending_epc_data, + fixed_data=fixed_data, + ) + + loft_scoring = create_recommendation_scoring_data( + property=p, + recommendation=loft_simulation, + starting_epc_data=starting_epc_data, + ending_epc_data=ending_epc_data, + fixed_data=fixed_data, + ) + + return [cavity_scoring, loft_scoring] def get_ha_32data(ha_data, cleaned, cleaning_data, created_at): diff --git a/etl/eligibility/ha_15_32/ha33_app.py b/etl/eligibility/ha_15_32/ha33_app.py index 1f8c15df..f5152fba 100644 --- a/etl/eligibility/ha_15_32/ha33_app.py +++ b/etl/eligibility/ha_15_32/ha33_app.py @@ -10,6 +10,10 @@ from tqdm import tqdm from backend.SearchEpc import SearchEpc from etl.eligibility.Eligibility import Eligibility from etl.eligibility.ha_15_32.app import prepare_model_data_row +from etl.epc.DataProcessor import DataProcessor +from etl.epc.settings import COLUMNS_TO_MERGE_ON +from backend.ml_models.api import ModelApi + import re ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env" @@ -79,6 +83,8 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at): flat_pattern = r'flat\s+(\d+)' + # data = data[data["row_id"].isin(eco_row_ids)] + scoring_data = [] results = [] nodata = [] @@ -125,7 +131,7 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at): cleaning_data=cleaning_data, created_at=created_at ) - scoring_data.append(scoring_dictionary) + scoring_data.extend(scoring_dictionary) # If nothing is eligible or gbis is eligible, then we make a record this results.append( @@ -155,8 +161,131 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at): # "scoring_data": scoring_data, # "nodata": nodata # }, f) + # with open("ha33_results.pickle", "rb") as f: + # data = pickle.load(f) + # results = data["results"] + # scoring_data = data["scoring_data"] + # nodata = data["nodata"] - return results, scoring_data, nodata + scoring_df = pd.DataFrame(scoring_data) + # Implement the same process that is being used in the recommendation engine to cleaning scoring_df + + # Perform the same cleaning as in the model - first clean number of room variables though + scoring_df = DataProcessor.apply_averages_cleaning( + data_to_clean=scoring_df, + cleaning_data=cleaning_data, + cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'], + colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"], + ) + + scoring_df = DataProcessor.apply_averages_cleaning( + data_to_clean=scoring_df, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"], + ).drop(columns=["LOCAL_AUTHORITY"]) + + scoring_df = DataProcessor.clean_missings_after_description_process( + scoring_df, + ignore_cols=[c for c in scoring_df.columns if ("thermal_transmittance" in c) or ( + "insulation_thickness" in c) or ("ENERGY_EFF" in c)] + ) + + scoring_df = DataProcessor.clean_efficiency_variables(scoring_df) + + model_api = ModelApi(portfolio_id="ha33-eligibility", timestamp=created_at) + all_predictions = model_api.predict_all( + df=scoring_df, + bucket="retrofit-data-dev", + prediction_buckets={ + "sap_change_predictions": "retrofit-sap-predictions-dev", + "heat_demand_predictions": "retrofit-heat-predictions-dev", + "carbon_change_predictions": "retrofit-carbon-predictions-dev" + } + ) + + # merge the predictions onto the scoring_df + predictions = all_predictions["sap_change_predictions"].copy() + + results_df = pd.DataFrame(results) + + predictions = predictions.rename(columns={"property_id": "row_id"}).merge( + results_df[["row_id", "sap"]], how="left", on="row_id" + ) + predictions["sap_uplift"] = predictions["predictions"] - predictions["sap"] + predictions = predictions.groupby("row_id")["sap_uplift"].sum().reset_index() + + results_df = results_df.merge( + predictions[["sap_uplift", "row_id"]], + how="left", + on="row_id" + ) + + results_df["post_install_sap"] = results_df["sap"] + results_df["sap_uplift"] + + eligibility_assessment = [] + for _, row in results_df[results_df["eco4_eligible"] == True].iterrows(): + # The upgrade requirements are dependent on the current SAP + + # If the property is an F or G, it only needs to upgrade to an % + if row["sap"] <= 38: + if row["post_install_sap"] >= 57: + eligibility_classification = "highest confidence" + elif row["post_install_sap"] >= 55: + eligibility_classification = "high confidence" + elif row["post_install_sap"] >= 53: + eligibility_classification = "medium confidence" + else: + eligibility_classification = "unlikely" + else: + + if row["post_install_sap"] >= 71: + eligibility_classification = "highest confidence" + elif row["post_install_sap"] >= 69: + eligibility_classification = "high confidence" + elif row["post_install_sap"] >= 67: + eligibility_classification = "medium confidence" + else: + eligibility_classification = "unlikely" + + eligibility_assessment.append( + { + "row_id": row["row_id"], + "eligibility_classification": eligibility_classification + } + ) + + eligibility_assessment = pd.DataFrame(eligibility_assessment) + + results_df = results_df.merge( + eligibility_assessment, how="left", on="row_id" + ) + + return results_df, scoring_data, nodata + + +def analyse_ha_33(results_df, data): + results_df_social = results_df[results_df["tenure"] == "Rented (social)"] + + results_df_social["tenure"].value_counts() + + data[data["row_id"].isin(results_df_social["row_id"].values)]["PROPERTY TYPE"].value_counts() + + n_identified = (results_df_social["gbis_eligible"] | results_df_social["eco4_eligible"]).sum() + n_eco4 = results_df_social["eco4_eligible"].sum() + n_gbis = results_df_social[~results_df_social["eco4_eligible"]]["gbis_eligible"].sum() + + results_df_social[results_df_social["gbis_eligible"] | results_df_social["eco4_eligible"]]["tenure"].value_counts() + + results_df_social["eligibility_classification"].value_counts() + + future_possibilities_eco = results_df[ + (results_df["eco4_eligible_future"] == True) & (~(results_df["gbis_eligible"] | results_df["eco4_eligible"])) + ].copy() + + future_possibilities_gbis = results_df[ + (results_df["gbis_eligible_future"] == True) & (results_df["eco4_eligible_future"] == False) & ( + ~(results_df["gbis_eligible"] | results_df["eco4_eligible"])) + ].copy() def app(): @@ -182,4 +311,4 @@ def app(): created_at = datetime.now().isoformat() - get_ha_33data(data, cleaned, cleaning_data, created_at) + results_df, _, _ = get_ha_33data(data, cleaned, cleaning_data, created_at)