From 2cb5308711279ded9986ecf22c61676f1124c3f3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 12 Aug 2024 10:32:26 +0100 Subject: [PATCH] cleaning setting of kwh and energy bills --- backend/app/plan/router.py | 229 +------------------------ backend/ml_models/AnnualBillSavings.py | 8 + etl/bill_savings/data_collection.py | 2 +- etl/bill_savings/training_data.py | 2 + recommendations/Recommendations.py | 218 +++++++++++++++++++++++ 5 files changed, 236 insertions(+), 223 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6dce42a9..53a6d813 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,6 +30,7 @@ from backend.app.plan.utils import get_cleaned from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc from backend.ml_models.api import ModelApi +from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.Property import Property from backend.apis.GoogleSolarApi import GoogleSolarApi @@ -722,228 +723,12 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations = recommendations[property_id] property_instance = [p for p in input_properties if p.id == property_id][0] - kwh_impact_table = kwh_simulation_predictions["heating_kwh_predictions"][ - kwh_simulation_predictions["heating_kwh_predictions"]["property_id"] == str(property_id) - ].merge( - kwh_simulation_predictions["hotwater_kwh_predictions"].drop( - columns=["property_id", "recommendation_id", "phase"] - ), - how="inner", - on="id", - suffixes=("_heating", "_hotwater") - ).reset_index(drop=True) - - # We adjust this table with the kwh estimates for low energy lighting kwh values, and solar kwh estimates - led_recommendation = pd.DataFrame([ - { - "phase": r["phase"], - "recommendation_id": r["recommendation_id"], - "lighting_kwh_savings": r["kwh_savings"] * GoogleSolarApi.SOLAR_CONSUMPTION_PROPORTION, - } for recs in property_recommendations for r in recs if r["type"] == "low_energy_lighting" - ], columns=["phase", "recommendation_id", "lighting_kwh_savings"]) - solar_recommendations = pd.DataFrame([ - { - "phase": r["phase"], - "recommendation_id": r["recommendation_id"], - "solar_kwh_savings": r["initial_ac_kwh_per_year"] * GoogleSolarApi.SOLAR_CONSUMPTION_PROPORTION, - } for recs in property_recommendations for r in recs if r["type"] == "solar_pv" - ], columns=["phase", "recommendation_id", "solar_kwh_savings"]) - - # merge them on - kwh_impact_table = kwh_impact_table.merge( - led_recommendation, how="left", on=["phase", "recommendation_id"] - ).merge( - solar_recommendations, how="left", on=["phase", "recommendation_id"] - ) - - property_kwh = property_instance.energy_consumption_estimates["unadjusted"] - - starting_dummy_id_value = -9999 - kwh_impact_table = pd.concat( - [ - pd.DataFrame( - [ - { - "id": starting_dummy_id_value, - "phase": starting_dummy_id_value, - "recommendation_id": starting_dummy_id_value, - "predictions_heating": property_kwh["heating"], - "predictions_hotwater": property_kwh["hot_water"], - } - ] - ), - kwh_impact_table - ] - ).sort_values(["phase", "recommendation_id"], ascending=True).reset_index(drop=True) - - for i in range(0, len(kwh_impact_table)): - current_phase = kwh_impact_table.loc[i, 'phase'] - previous_phase_id = (current_phase - 1) if (current_phase > 0) else -9999 - previous_phase = kwh_impact_table[kwh_impact_table['phase'] == previous_phase_id] - - if not previous_phase.empty: - for col in ["predictions_heating", "predictions_hotwater"]: - if kwh_impact_table.loc[i, col] > previous_phase[col].max(): - kwh_impact_table.loc[i, col] = previous_phase[col].max() - - from backend.ml_models.AnnualBillSavings import AnnualBillSavings - # We adjust the predictions with the UCL model - for k in ["heating", "hotwater"]: - kwh_impact_table[f"adjusted_{k}"] = kwh_impact_table[f"predictions_{k}"].apply( - lambda x: AnnualBillSavings.adjust_energy_to_metered( - epc_energy=x, current_epc_rating=property_instance.data["current-energy-rating"] - ) - ) - - ASHP_COP = 3 - descriptions_to_fuel_types = { - "Air source heat pump, radiators, electric": {"fuel": "Electricity", "cop": ASHP_COP}, - "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - 'Electric storage heaters': {"fuel": 'Electricity', "cop": 1}, - "Electric immersion, off-peak": {"fuel": 'Electricity', "cop": 1}, - "Electric storage heaters, radiators": {"fuel": 'Electricity', "cop": 1}, - "Room heaters, electric": {"fuel": 'Electricity', "cop": 1}, - "Electric immersion, standard tariff": {"fuel": 'Electricity', "cop": 1}, - "Portable electric heaters assumed for most rooms": {"fuel": 'Electricity', "cop": 1}, - } - - def map_descriptions_to_fuel(heating_description, hotwater_description): - mapped = descriptions_to_fuel_types[heating_description] - heating_fuel = mapped["fuel"] - - if hotwater_description == "From main system": - return { - "heating_fuel_type": heating_fuel, "hotwater_fuel_type": heating_fuel, - "heating_cop": mapped["cop"], "hotwater_cop": mapped["cop"] - } - - mapped_hotwater = descriptions_to_fuel_types[hotwater_description] - - return { - "heating_fuel_type": heating_fuel, "hotwater_fuel_type": mapped_hotwater["fuel"], - "heating_cop": mapped["cop"], "hotwater_cop": mapped_hotwater["cop"] - } - - # For heating system recommendations, this could result in a fuel type change so we reflect that - fuel_mapping = pd.DataFrame([ - { - "id": epc["id"], - **map_descriptions_to_fuel(epc["mainheat-description"], epc["hotwater-description"]) - } for epc in property_instance.updated_simulation_epcs - ]) - - for epc in property_instance.updated_simulation_epcs: - map_descriptions_to_fuel(epc["mainheat-description"], epc["hotwater-description"]) - - fuel_mapping = pd.concat( - [ - pd.DataFrame( - [ - { - "id": starting_dummy_id_value, - **map_descriptions_to_fuel( - property_instance.data["mainheat-description"], - property_instance.data["hotwater-description"] - ) - } - ] - ), - fuel_mapping - ] - ) - - kwh_impact_table = kwh_impact_table.merge( - fuel_mapping, how="left", on="id" - ).sort_values(["phase", "recommendation_id"], ascending=True).reset_index(drop=True) - - kwh_impact_table["heating_fuel_type"] = np.where( - kwh_impact_table["id"] == starting_dummy_id_value, - property_instance.heating_energy_source, - kwh_impact_table["heating_fuel_type"] - ) - - kwh_impact_table["hotwater_fuel_type"] = np.where( - kwh_impact_table["id"] == starting_dummy_id_value, - property_instance.hot_water_energy_source, - kwh_impact_table["hotwater_fuel_type"] - ) - - def calculate_recommendation_fuel_cost(kwh, fuel, cop): - if fuel == "Electricity": - return (kwh / cop) * AnnualBillSavings.ELECTRICITY_PRICE_CAP - - if fuel == "Natural Gas": - return (kwh / cop) * AnnualBillSavings.GAS_PRICE_CAP - - # We now calculate the fuel cost - for k in ["heating", "hotwater"]: - kwh_impact_table[f"{k}_cost"] = kwh_impact_table.apply( - lambda x: calculate_recommendation_fuel_cost( - x[f"adjusted_{k}"], x[f"{k}_fuel_type"], x[f"{k}_cop"] - ), axis=1 - ) - - # TODO: The impact of remapping EPC is huge! - - # We now deduce if any of the recommendations result in a change of fuel type - for recs in property_recommendations: - for rec in recs: - if rec["type"] == "mechanical_ventilation": - continue - - rec_impact = kwh_impact_table[kwh_impact_table["recommendation_id"] == rec["recommendation_id"]] - prevous_phase_id = (rec["phase"] - 1) if (rec["phase"] > 0) else starting_dummy_id_value - previous_phase_impact = kwh_impact_table[kwh_impact_table["phase"] == prevous_phase_id] - - if rec["type"] == "solar_pv": - rec["kwh_savings"] = rec_impact["solar_kwh_savings"].values[0] - rec["energy_cost_savings"] = ( - rec_impact["solar_kwh_savings"].values[0] * AnnualBillSavings.ELECTRICITY_PRICE_CAP - ) - continue - - heating_kwh_savings = ( - previous_phase_impact["adjusted_heating"].mean() - rec_impact["adjusted_heating"].values[0] - ) - heating_cost_savings = ( - previous_phase_impact["heating_cost"].mean() - rec_impact["heating_cost"].values[0] - ) - - hotwater_kwh_savings = ( - previous_phase_impact["adjusted_hotwater"].mean() - rec_impact["adjusted_hotwater"].values[0] - ) - hotwater_host = ( - previous_phase_impact["hotwater_cost"].mean() - rec_impact["hotwater_cost"].values[0] - ) - - total_kwh_savings = heating_kwh_savings + hotwater_kwh_savings - energy_cost_savings = heating_cost_savings + hotwater_host - - if rec["type"] == "lighting": - # In this case, we should probably just SKIP but check when we have one! - raise Exception("Implement me 3") - - rec["kwh_savings"] = total_kwh_savings - rec["energy_cost_savings"] = energy_cost_savings - - # Finally, we set the current energy bill - starting_figures = kwh_impact_table[kwh_impact_table["id"] == starting_dummy_id_value].squeeze() - gas_standing_charge = 0 - if ( - (starting_figures["heating_fuel_type"] == "Natural Gas") or - (starting_figures["hotwater_fuel_type"] == "Natural Gas") - ): - gas_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_GAS * 365 - - electricity_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_ELECTRICITY * 365 - - property_instance.current_energy_bill = ( - starting_figures["heating_cost"].values[0] + - starting_figures["hotwater_cost"].values[0] + - property_instance.energy_cost_estimates["unadjusted"]["lighting"] + - property_instance.energy_cost_estimates["unadjusted"]["appliances"] + - gas_standing_charge + electricity_standing_charge + property_current_energy_bill = Recommendations.calculate_recommendation_tenant_savings( + property_instance=property_instance, + kwh_simulation_predictions=kwh_simulation_predictions, + property_recommendations=property_recommendations ) + property_instance.current_energy_bill = property_current_energy_bill # Insert the predictions into the recommendations and run the optimiser # TODO: If a recommendation has a negative impact on SAP, we should remove it - this seems to have become a @@ -1284,7 +1069,7 @@ async def build_mds(body: MdsRequest): recommendations = {} for p in tqdm(input_properties): - p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) + p.set_features(cleaned, photo_supply_lookup, floor_area_decile_thresholds) mds = Mds(property_instance=p, materials=materials, optimise_measures=optimise_measures) mds_recommendations, property_representative_recommendations, errors = mds.build() diff --git a/backend/ml_models/AnnualBillSavings.py b/backend/ml_models/AnnualBillSavings.py index e4d9d143..0317b9e3 100644 --- a/backend/ml_models/AnnualBillSavings.py +++ b/backend/ml_models/AnnualBillSavings.py @@ -199,3 +199,11 @@ class AnnualBillSavings: return current_epc_rating return cls.EPC_BANDS[expected_index - 1] + + @classmethod + def calculate_recommendation_fuel_cost(cls, kwh, fuel, cop): + if fuel == "Electricity": + return (kwh / cop) * cls.ELECTRICITY_PRICE_CAP + + if fuel == "Natural Gas": + return (kwh / cop) * cls.GAS_PRICE_CAP diff --git a/etl/bill_savings/data_collection.py b/etl/bill_savings/data_collection.py index a073a70e..75fd9df2 100644 --- a/etl/bill_savings/data_collection.py +++ b/etl/bill_savings/data_collection.py @@ -134,7 +134,7 @@ def app(): for i, directory in tqdm(enumerate(epc_directories), total=len(epc_directories)): try: # Skip the first 50 - if i < 40: + if i < 200: continue data = pd.read_csv(directory / "certificates.csv", low_memory=False) diff --git a/etl/bill_savings/training_data.py b/etl/bill_savings/training_data.py index 85b53bca..a3d58af3 100644 --- a/etl/bill_savings/training_data.py +++ b/etl/bill_savings/training_data.py @@ -17,6 +17,8 @@ def app(): cleaned = msgpack.unpackb(cleaned, raw=False) + # If there is any problematic data, it could be: + # s3://retrofit-datalake-dev/energy_consumption_data/2024-08-10 18:48:06.866647.pkl kwh_data_client = KwhData(bucket="retrofit-datalake-dev") kwh_data_client.combine() kwh_data_client.transform(data=kwh_data_client.data, cleaned=cleaned, save=True) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 588d2316..21c4f551 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -17,6 +17,19 @@ from recommendations.SecondaryHeating import SecondaryHeating from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.apis.GoogleSolarApi import GoogleSolarApi +ASHP_COP = 3 +DESCRIPTIONS_TO_FUEL_TYPES = { + "Air source heat pump, radiators, electric": {"fuel": "Electricity", "cop": ASHP_COP}, + "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, + 'Electric storage heaters': {"fuel": 'Electricity', "cop": 1}, + "Electric immersion, off-peak": {"fuel": 'Electricity', "cop": 1}, + "Electric storage heaters, radiators": {"fuel": 'Electricity', "cop": 1}, + "Room heaters, electric": {"fuel": 'Electricity', "cop": 1}, + "Electric immersion, standard tariff": {"fuel": 'Electricity', "cop": 1}, + "Portable electric heaters assumed for most rooms": {"fuel": 'Electricity', "cop": 1}, +} +STARTING_DUMMY_ID_VALUE = -9999 + class Recommendations: """ @@ -497,3 +510,208 @@ class Recommendations: ) return property_recommendations, impact_summary + + @staticmethod + def map_descriptions_to_fuel(heating_description, hotwater_description): + mapped = DESCRIPTIONS_TO_FUEL_TYPES[heating_description] + heating_fuel = mapped["fuel"] + + if hotwater_description == "From main system": + return { + "heating_fuel_type": heating_fuel, "hotwater_fuel_type": heating_fuel, + "heating_cop": mapped["cop"], "hotwater_cop": mapped["cop"] + } + + mapped_hotwater = DESCRIPTIONS_TO_FUEL_TYPES[hotwater_description] + + return { + "heating_fuel_type": heating_fuel, "hotwater_fuel_type": mapped_hotwater["fuel"], + "heating_cop": mapped["cop"], "hotwater_cop": mapped_hotwater["cop"] + } + + @classmethod + def calculate_recommendation_tenant_savings( + cls, property_instance, kwh_simulation_predictions, property_recommendations + ): + """ + This method inserts the kwh savings and the bill savings that the customer will make from the recommendations + based on the predictions from the ML model + :param property_instance: Instance of the Property class, for the home associated to property_id + :param kwh_simulation_predictions: dictionary of predictions from the model apis + :param property_recommendations: dictionary of recommendations for the property + :return: + """ + + kwh_impact_table = kwh_simulation_predictions["heating_kwh_predictions"][ + kwh_simulation_predictions["heating_kwh_predictions"]["property_id"] == str(property_instance.id) + ].merge( + kwh_simulation_predictions["hotwater_kwh_predictions"].drop( + columns=["property_id", "recommendation_id", "phase"] + ), + how="inner", + on="id", + suffixes=("_heating", "_hotwater") + ).reset_index(drop=True) + + # We adjust this table with the kwh estimates for low energy lighting kwh values, and solar kwh estimates + led_recommendation = pd.DataFrame([ + { + "phase": r["phase"], + "recommendation_id": r["recommendation_id"], + "lighting_kwh_savings": r["kwh_savings"] * GoogleSolarApi.SOLAR_CONSUMPTION_PROPORTION, + } for recs in property_recommendations for r in recs if r["type"] == "low_energy_lighting" + ], columns=["phase", "recommendation_id", "lighting_kwh_savings"]) + solar_recommendations = pd.DataFrame([ + { + "phase": r["phase"], + "recommendation_id": r["recommendation_id"], + "solar_kwh_savings": r["initial_ac_kwh_per_year"] * GoogleSolarApi.SOLAR_CONSUMPTION_PROPORTION, + } for recs in property_recommendations for r in recs if r["type"] == "solar_pv" + ], columns=["phase", "recommendation_id", "solar_kwh_savings"]) + + # merge them on + kwh_impact_table = kwh_impact_table.merge( + led_recommendation, how="left", on=["phase", "recommendation_id"] + ).merge( + solar_recommendations, how="left", on=["phase", "recommendation_id"] + ) + + property_kwh = property_instance.energy_consumption_estimates["unadjusted"] + + kwh_impact_table = pd.concat( + [ + pd.DataFrame( + [ + { + "id": STARTING_DUMMY_ID_VALUE, + "phase": STARTING_DUMMY_ID_VALUE, + "recommendation_id": STARTING_DUMMY_ID_VALUE, + "predictions_heating": property_kwh["heating"], + "predictions_hotwater": property_kwh["hot_water"], + } + ] + ), + kwh_impact_table + ] + ).sort_values(["phase", "recommendation_id"], ascending=True).reset_index(drop=True) + + for i in range(0, len(kwh_impact_table)): + current_phase = kwh_impact_table.loc[i, 'phase'] + previous_phase_id = (current_phase - 1) if (current_phase > 0) else -9999 + previous_phase = kwh_impact_table[kwh_impact_table['phase'] == previous_phase_id] + + if not previous_phase.empty: + for col in ["predictions_heating", "predictions_hotwater"]: + if kwh_impact_table.loc[i, col] > previous_phase[col].max(): + kwh_impact_table.loc[i, col] = previous_phase[col].max() + + # For heating system recommendations, this could result in a fuel type change so we reflect that + fuel_mapping = pd.DataFrame([ + { + "id": epc["id"], + **cls.map_descriptions_to_fuel(epc["mainheat-description"], epc["hotwater-description"]) + } for epc in property_instance.updated_simulation_epcs + ]) + + fuel_mapping = pd.concat( + [ + pd.DataFrame( + [ + { + "id": STARTING_DUMMY_ID_VALUE, + **cls.map_descriptions_to_fuel( + property_instance.data["mainheat-description"], + property_instance.data["hotwater-description"] + ) + } + ] + ), + fuel_mapping + ] + ) + + kwh_impact_table = kwh_impact_table.merge( + fuel_mapping, how="left", on="id" + ).sort_values(["phase", "recommendation_id"], ascending=True).reset_index(drop=True) + + kwh_impact_table["heating_fuel_type"] = np.where( + kwh_impact_table["id"] == STARTING_DUMMY_ID_VALUE, + property_instance.heating_energy_source, + kwh_impact_table["heating_fuel_type"] + ) + + kwh_impact_table["hotwater_fuel_type"] = np.where( + kwh_impact_table["id"] == STARTING_DUMMY_ID_VALUE, + property_instance.hot_water_energy_source, + kwh_impact_table["hotwater_fuel_type"] + ) + + # We now calculate the fuel cost + for k in ["heating", "hotwater"]: + kwh_impact_table[f"{k}_cost"] = kwh_impact_table.apply( + lambda x: AnnualBillSavings.calculate_recommendation_fuel_cost( + x[f"adjusted_{k}"], x[f"{k}_fuel_type"], x[f"{k}_cop"] + ), axis=1 + ) + + # We now deduce if any of the recommendations result in a change of fuel type + for recs in property_recommendations: + for rec in recs: + if rec["type"] == "mechanical_ventilation": + continue + + rec_impact = kwh_impact_table[kwh_impact_table["recommendation_id"] == rec["recommendation_id"]] + prevous_phase_id = (rec["phase"] - 1) if (rec["phase"] > 0) else STARTING_DUMMY_ID_VALUE + previous_phase_impact = kwh_impact_table[kwh_impact_table["phase"] == prevous_phase_id] + + if rec["type"] == "solar_pv": + rec["kwh_savings"] = rec_impact["solar_kwh_savings"].values[0] + rec["energy_cost_savings"] = ( + rec_impact["solar_kwh_savings"].values[0] * AnnualBillSavings.ELECTRICITY_PRICE_CAP + ) + continue + + heating_kwh_savings = ( + previous_phase_impact["adjusted_heating"].mean() - rec_impact["adjusted_heating"].values[0] + ) + heating_cost_savings = ( + previous_phase_impact["heating_cost"].mean() - rec_impact["heating_cost"].values[0] + ) + + hotwater_kwh_savings = ( + previous_phase_impact["adjusted_hotwater"].mean() - rec_impact["adjusted_hotwater"].values[0] + ) + hotwater_host = ( + previous_phase_impact["hotwater_cost"].mean() - rec_impact["hotwater_cost"].values[0] + ) + + total_kwh_savings = heating_kwh_savings + hotwater_kwh_savings + energy_cost_savings = heating_cost_savings + hotwater_host + + if rec["type"] == "lighting": + # In this case, we should probably just SKIP but check when we have one! + raise Exception("Implement me 3") + + rec["kwh_savings"] = total_kwh_savings + rec["energy_cost_savings"] = energy_cost_savings + + # Finally, we set the current energy bill + starting_figures = kwh_impact_table[kwh_impact_table["id"] == STARTING_DUMMY_ID_VALUE].squeeze() + gas_standing_charge = 0 + if ( + (starting_figures["heating_fuel_type"] == "Natural Gas") or + (starting_figures["hotwater_fuel_type"] == "Natural Gas") + ): + gas_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_GAS * 365 + + electricity_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_ELECTRICITY * 365 + + current_energy_bill = ( + starting_figures["heating_cost"].values[0] + + starting_figures["hotwater_cost"].values[0] + + property_instance.energy_cost_estimates["unadjusted"]["lighting"] + + property_instance.energy_cost_estimates["unadjusted"]["appliances"] + + gas_standing_charge + electricity_standing_charge + ) + + return current_energy_bill