mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
done with ha33 analysis for now
This commit is contained in:
Khalim Conn-Kowlessar
parent
01d8e52650
commit
6ab3804b4f
2 changed files with 162 additions and 32 deletions
|
|
@ -376,37 +376,38 @@ def prepare_model_data_row(property_id, modelling_epc, cleaned, cleaning_data, c
|
|||
# after retrofit. We use the minimal u-values required to meet building regulations part L
|
||||
# TODO: Check the performance of the materials warmfront's installers use, particularly for
|
||||
# cavity
|
||||
simulation_recommendations = [
|
||||
{
|
||||
"recommendation_id": "-".join([property_id, "cavity"]),
|
||||
"type": "cavity_wall_insulation",
|
||||
"new_u_value": 0.35,
|
||||
"parts": [{}]
|
||||
},
|
||||
{
|
||||
"recommendation_id": "-".join([property_id, "loft"]),
|
||||
"type": "loft_insulation",
|
||||
"new_u_value": 0.16,
|
||||
"parts": [{"depth": 270}]
|
||||
}
|
||||
]
|
||||
|
||||
scoring_dict = {}
|
||||
for recommendation in simulation_recommendations:
|
||||
scoring_dict = create_recommendation_scoring_data(
|
||||
property=p,
|
||||
recommendation=recommendation,
|
||||
starting_epc_data=starting_epc_data,
|
||||
ending_epc_data=ending_epc_data,
|
||||
fixed_data=fixed_data,
|
||||
)
|
||||
# At each iteration, we want to update the ending_epc_data, so in the end, ending_epc_data contains
|
||||
# all of the updates
|
||||
for k in scoring_dict.keys():
|
||||
if k in ending_epc_data.columns:
|
||||
ending_epc_data[k] = scoring_dict[k]
|
||||
cavity_simulation = {
|
||||
"recommendation_id": "-".join([property_id, "cavity"]),
|
||||
"type": "cavity_wall_insulation",
|
||||
"new_u_value": 0.35,
|
||||
"parts": [{}]
|
||||
}
|
||||
|
||||
return scoring_dict
|
||||
loft_simulation = {
|
||||
"recommendation_id": "-".join([property_id, "loft"]),
|
||||
"type": "loft_insulation",
|
||||
"new_u_value": 0.16,
|
||||
"parts": [{"depth": 270}]
|
||||
}
|
||||
|
||||
cavity_scoring = create_recommendation_scoring_data(
|
||||
property=p,
|
||||
recommendation=cavity_simulation,
|
||||
starting_epc_data=starting_epc_data,
|
||||
ending_epc_data=ending_epc_data,
|
||||
fixed_data=fixed_data,
|
||||
)
|
||||
|
||||
loft_scoring = create_recommendation_scoring_data(
|
||||
property=p,
|
||||
recommendation=loft_simulation,
|
||||
starting_epc_data=starting_epc_data,
|
||||
ending_epc_data=ending_epc_data,
|
||||
fixed_data=fixed_data,
|
||||
)
|
||||
|
||||
return [cavity_scoring, loft_scoring]
|
||||
|
||||
|
||||
def get_ha_32data(ha_data, cleaned, cleaning_data, created_at):
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ from tqdm import tqdm
|
|||
from backend.SearchEpc import SearchEpc
|
||||
from etl.eligibility.Eligibility import Eligibility
|
||||
from etl.eligibility.ha_15_32.app import prepare_model_data_row
|
||||
from etl.epc.DataProcessor import DataProcessor
|
||||
from etl.epc.settings import COLUMNS_TO_MERGE_ON
|
||||
from backend.ml_models.api import ModelApi
|
||||
|
||||
import re
|
||||
|
||||
ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"
|
||||
|
|
@ -79,6 +83,8 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at):
|
|||
|
||||
flat_pattern = r'flat\s+(\d+)'
|
||||
|
||||
# data = data[data["row_id"].isin(eco_row_ids)]
|
||||
|
||||
scoring_data = []
|
||||
results = []
|
||||
nodata = []
|
||||
|
|
@ -125,7 +131,7 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at):
|
|||
cleaning_data=cleaning_data,
|
||||
created_at=created_at
|
||||
)
|
||||
scoring_data.append(scoring_dictionary)
|
||||
scoring_data.extend(scoring_dictionary)
|
||||
|
||||
# If nothing is eligible or gbis is eligible, then we make a record this
|
||||
results.append(
|
||||
|
|
@ -155,8 +161,131 @@ def get_ha_33data(data, cleaned, cleaning_data, created_at):
|
|||
# "scoring_data": scoring_data,
|
||||
# "nodata": nodata
|
||||
# }, f)
|
||||
# with open("ha33_results.pickle", "rb") as f:
|
||||
# data = pickle.load(f)
|
||||
# results = data["results"]
|
||||
# scoring_data = data["scoring_data"]
|
||||
# nodata = data["nodata"]
|
||||
|
||||
return results, scoring_data, nodata
|
||||
scoring_df = pd.DataFrame(scoring_data)
|
||||
# Implement the same process that is being used in the recommendation engine to cleaning scoring_df
|
||||
|
||||
# Perform the same cleaning as in the model - first clean number of room variables though
|
||||
scoring_df = DataProcessor.apply_averages_cleaning(
|
||||
data_to_clean=scoring_df,
|
||||
cleaning_data=cleaning_data,
|
||||
cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'],
|
||||
colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"],
|
||||
)
|
||||
|
||||
scoring_df = DataProcessor.apply_averages_cleaning(
|
||||
data_to_clean=scoring_df,
|
||||
cleaning_data=cleaning_data,
|
||||
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"],
|
||||
).drop(columns=["LOCAL_AUTHORITY"])
|
||||
|
||||
scoring_df = DataProcessor.clean_missings_after_description_process(
|
||||
scoring_df,
|
||||
ignore_cols=[c for c in scoring_df.columns if ("thermal_transmittance" in c) or (
|
||||
"insulation_thickness" in c) or ("ENERGY_EFF" in c)]
|
||||
)
|
||||
|
||||
scoring_df = DataProcessor.clean_efficiency_variables(scoring_df)
|
||||
|
||||
model_api = ModelApi(portfolio_id="ha33-eligibility", timestamp=created_at)
|
||||
all_predictions = model_api.predict_all(
|
||||
df=scoring_df,
|
||||
bucket="retrofit-data-dev",
|
||||
prediction_buckets={
|
||||
"sap_change_predictions": "retrofit-sap-predictions-dev",
|
||||
"heat_demand_predictions": "retrofit-heat-predictions-dev",
|
||||
"carbon_change_predictions": "retrofit-carbon-predictions-dev"
|
||||
}
|
||||
)
|
||||
|
||||
# merge the predictions onto the scoring_df
|
||||
predictions = all_predictions["sap_change_predictions"].copy()
|
||||
|
||||
results_df = pd.DataFrame(results)
|
||||
|
||||
predictions = predictions.rename(columns={"property_id": "row_id"}).merge(
|
||||
results_df[["row_id", "sap"]], how="left", on="row_id"
|
||||
)
|
||||
predictions["sap_uplift"] = predictions["predictions"] - predictions["sap"]
|
||||
predictions = predictions.groupby("row_id")["sap_uplift"].sum().reset_index()
|
||||
|
||||
results_df = results_df.merge(
|
||||
predictions[["sap_uplift", "row_id"]],
|
||||
how="left",
|
||||
on="row_id"
|
||||
)
|
||||
|
||||
results_df["post_install_sap"] = results_df["sap"] + results_df["sap_uplift"]
|
||||
|
||||
eligibility_assessment = []
|
||||
for _, row in results_df[results_df["eco4_eligible"] == True].iterrows():
|
||||
# The upgrade requirements are dependent on the current SAP
|
||||
|
||||
# If the property is an F or G, it only needs to upgrade to an %
|
||||
if row["sap"] <= 38:
|
||||
if row["post_install_sap"] >= 57:
|
||||
eligibility_classification = "highest confidence"
|
||||
elif row["post_install_sap"] >= 55:
|
||||
eligibility_classification = "high confidence"
|
||||
elif row["post_install_sap"] >= 53:
|
||||
eligibility_classification = "medium confidence"
|
||||
else:
|
||||
eligibility_classification = "unlikely"
|
||||
else:
|
||||
|
||||
if row["post_install_sap"] >= 71:
|
||||
eligibility_classification = "highest confidence"
|
||||
elif row["post_install_sap"] >= 69:
|
||||
eligibility_classification = "high confidence"
|
||||
elif row["post_install_sap"] >= 67:
|
||||
eligibility_classification = "medium confidence"
|
||||
else:
|
||||
eligibility_classification = "unlikely"
|
||||
|
||||
eligibility_assessment.append(
|
||||
{
|
||||
"row_id": row["row_id"],
|
||||
"eligibility_classification": eligibility_classification
|
||||
}
|
||||
)
|
||||
|
||||
eligibility_assessment = pd.DataFrame(eligibility_assessment)
|
||||
|
||||
results_df = results_df.merge(
|
||||
eligibility_assessment, how="left", on="row_id"
|
||||
)
|
||||
|
||||
return results_df, scoring_data, nodata
|
||||
|
||||
|
||||
def analyse_ha_33(results_df, data):
|
||||
results_df_social = results_df[results_df["tenure"] == "Rented (social)"]
|
||||
|
||||
results_df_social["tenure"].value_counts()
|
||||
|
||||
data[data["row_id"].isin(results_df_social["row_id"].values)]["PROPERTY TYPE"].value_counts()
|
||||
|
||||
n_identified = (results_df_social["gbis_eligible"] | results_df_social["eco4_eligible"]).sum()
|
||||
n_eco4 = results_df_social["eco4_eligible"].sum()
|
||||
n_gbis = results_df_social[~results_df_social["eco4_eligible"]]["gbis_eligible"].sum()
|
||||
|
||||
results_df_social[results_df_social["gbis_eligible"] | results_df_social["eco4_eligible"]]["tenure"].value_counts()
|
||||
|
||||
results_df_social["eligibility_classification"].value_counts()
|
||||
|
||||
future_possibilities_eco = results_df[
|
||||
(results_df["eco4_eligible_future"] == True) & (~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
|
||||
].copy()
|
||||
|
||||
future_possibilities_gbis = results_df[
|
||||
(results_df["gbis_eligible_future"] == True) & (results_df["eco4_eligible_future"] == False) & (
|
||||
~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
|
||||
].copy()
|
||||
|
||||
|
||||
def app():
|
||||
|
|
@ -182,4 +311,4 @@ def app():
|
|||
|
||||
created_at = datetime.now().isoformat()
|
||||
|
||||
get_ha_33data(data, cleaned, cleaning_data, created_at)
|
||||
results_df, _, _ = get_ha_33data(data, cleaned, cleaning_data, created_at)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue