From 456d0892c056a9b1728e438bca6d87154943ddbf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 9 Feb 2024 17:16:54 +0000 Subject: [PATCH] tweaks to router + added tenure filter to epc pipeline --- backend/Property.py | 5 +- backend/app/plan/router.py | 84 ++++++++++------------ backend/ml_models/AnnualBillSavings.py | 9 +++ etl/epc/DataProcessor.py | 7 ++ etl/epc/settings.py | 4 ++ etl/testing_data/retrofitted_properties.py | 6 ++ recommendations/LightingRecommendations.py | 32 +++++++++ recommendations/Recommendations.py | 14 ++-- 8 files changed, 108 insertions(+), 53 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 6d534689..de87099b 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -192,8 +192,9 @@ class Property: recommendation_record["walls_insulation_thickness_ending"] = "none" # Update description to indicate it's insulate - if recommendation["type"] in ["solid_floor_insulation", "suspended_floor_insulation", - "exposed_floor_insulation"]: + if recommendation["type"] in [ + "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation" + ]: if len(recommendation["parts"]) > 1: raise NotImplementedError("Have more than 1 floor insulation part - handle this case") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index b3d1c623..89c5187b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -164,6 +164,8 @@ async def trigger_plan(body: PlanTriggerRequest): model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at) + # recommendations_scoring_data.loc[17, 'photo_supply_ending'] = 50 + all_predictions = model_api.predict_all( df=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET, @@ -237,9 +239,10 @@ async def trigger_plan(body: PlanTriggerRequest): ] recommendations[property_id] = final_recommendations + # impact_df = pd.DataFrame(recommendations[input_properties[0].id]) + # This is a temporary step, to estimate the impact of the measured on heat demand and carbon # TODO: This needs to be cleaned up, if it happens to be kept - combined_recommendations_scoring_data = [] representative_recs = {} for property_id, property_recommendations in recommendations.items(): default_recommendations = [r for r in property_recommendations if r["default"]] @@ -276,58 +279,26 @@ async def trigger_plan(body: PlanTriggerRequest): representative_recs[property_id] = default_recommendations - property_instance = [p for p in input_properties if p.id == property_id][0] - - recommendation_record = property_instance.base_difference_record.df.to_dict("records")[0].copy() - - scoring_dict = {} - for rec in default_recommendations: - scoring_dict = Property.create_recommendation_scoring_data( - property_id=property_instance.id, - recommendation_record=recommendation_record, - recommendation=rec - ) - # At each iterations, we update the recommendation record with the changes reflectecd in the - # scoring_dict - for k in scoring_dict.keys(): - if k in recommendation_record.keys(): - recommendation_record[k] = scoring_dict[k] - - combined_recommendations_scoring_data.append(scoring_dict) - - # PERFORM SAME STEPS AGAIN - TODO: TO BE REMOVED - combined_recommendations_scoring_data = pd.DataFrame(combined_recommendations_scoring_data) - - all_combined_predictions = model_api.predict_all( - df=combined_recommendations_scoring_data, - bucket=get_settings().DATA_BUCKET, - prediction_buckets={ - "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, - "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, - "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET - } - ) - # We update the carbon and heat demand predictions + # TODO: The api call producing all_combined_predictions has been removed so we can potentially completely + # refactor this block to just perform the energy adjustments for property_id, property_recommendations in recommendations.items(): - combined_heat_demand = all_combined_predictions["heat_demand_predictions"] - combined_heat_demand = combined_heat_demand[combined_heat_demand["property_id"] == str(property_id)] - - combined_carbon = all_combined_predictions["carbon_change_predictions"] - combined_carbon = combined_carbon[combined_carbon["property_id"] == str(property_id)] property_instance = [p for p in input_properties if p.id == property_id][0] - carbon_change = float( - property_instance.data["co2-emissions-current"] - ) - combined_carbon["predictions"].values[0] + heat_demand_change = sum( + x.get("heat_demand", 0) for x in representative_recs[property_id] if + x["type"] not in ["mechanical_ventilation", "low_energy_lighting"] + ) + carbon_change = sum( + x.get("co2_equivalent_savings", 0) for x in representative_recs[property_id] if + x["type"] not in ["mechanical_ventilation", "low_energy_lighting"] + ) starting_heat_demand = ( float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area ) - expected_heat_demand = starting_heat_demand - ( - combined_heat_demand["predictions"].values[0] * property_instance.floor_area - ) + expected_heat_demand = starting_heat_demand - heat_demand_change # We don't want to adjust the heat demand for mechanical ventilation so we add it back on @@ -361,6 +332,21 @@ async def trigger_plan(body: PlanTriggerRequest): in representative_recs[property_id] ] representative_rec_data = pd.DataFrame(representative_rec_data) + # Suppress mechanical ventilation to have zero heat demand and co2 + representative_rec_data.loc[ + representative_rec_data["type"] == "mechanical_ventilation", "co2_equivalent_savings" + ] = 0 + representative_rec_data.loc[ + representative_rec_data["type"] == "mechanical_ventilation", "heat_demand" + ] = 0 + # Supress low energy lighting to have zero heat demand and co2 - this does not get affected by this process + representative_rec_data.loc[ + representative_rec_data["type"] == "low_energy_lighting", "co2_equivalent_savings" + ] = 0 + representative_rec_data.loc[ + representative_rec_data["type"] == "low_energy_lighting", "heat_demand" + ] = 0 + # Convert co2 and heat demand to proportions of their column sums representative_rec_data["co2_equivalent_savings_percent"] = ( representative_rec_data["co2_equivalent_savings"] / @@ -393,10 +379,18 @@ async def trigger_plan(body: PlanTriggerRequest): rec["co2_equivalent_savings"] = 0 rec["heat_demand"] = 0 rec["energy_cost_savings"] = 0 + elif rec["type"] == "low_energy_lighting": + # We do not convert, we just calculate energy cost savings + rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"]) + continue else: rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0] rec["heat_demand"] = change_data["heat_demand"].values[0] - rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"]) + # If the recommendation is solar, the savings are entirely in electricity + if rec["type"] == "solar_pv": + rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"]) + else: + rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"]) # Update recommendations recommendations[property_id] = property_recommendations diff --git a/backend/ml_models/AnnualBillSavings.py b/backend/ml_models/AnnualBillSavings.py index f3e5074a..5e65cc5c 100644 --- a/backend/ml_models/AnnualBillSavings.py +++ b/backend/ml_models/AnnualBillSavings.py @@ -29,6 +29,15 @@ class AnnualBillSavings: """ return cls.PRICE_FACTOR * kwh + @classmethod + def estimate_electric(cls, kwh: float): + """ + Estimate the annual bill savings based on the kwh savings + :param kwh: The kwh savings + :return: An estimate for annual bill savings + """ + return cls.ELECTRICITY_PRICE_CAP * kwh + @classmethod def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating): """ diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 4615d2c4..5a210d9f 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -8,6 +8,7 @@ from etl.epc.settings import ( IGNORED_TRANSACTION_TYPES, IGNORED_FLOOR_LEVELS, IGNORED_PROPERTY_TYPES, + IGNORED_TENURES, FULLY_GLAZED_DESCRIPTIONS, AVERAGE_FIXED_FEATURES, BUILT_FORM_REMAP, @@ -632,6 +633,7 @@ class EPCDataProcessor: violation_missing_hotwater_description = pd.isnull(self.data["HOTWATER_DESCRIPTION"]) violation_missing_roof_description = pd.isnull(self.data["ROOF_DESCRIPTION"]) violation_invalid_property_type = self.data["PROPERTY_TYPE"] == IGNORED_PROPERTY_TYPES + violation_invalid_tenure = self.data["TENURE"].isin(IGNORED_TENURES) violation_df = pd.concat( [ @@ -644,6 +646,7 @@ class EPCDataProcessor: violation_missing_hotwater_description, violation_missing_roof_description, violation_invalid_property_type, + violation_invalid_tenure, ], axis=1, keys=[ "violation_uprn_missing", @@ -655,6 +658,7 @@ class EPCDataProcessor: "violation_missing_hotwater_description", "violation_missing_roof_description", "violation_invalid_property_type", + "violation_invalid_tenure" ] ) @@ -697,6 +701,9 @@ class EPCDataProcessor: # EPCs) we'll ignore them from the model self.data = self.data[self.data["PROPERTY_TYPE"] != IGNORED_PROPERTY_TYPES] + # We remove EPCs where the tenure is unknown, but is usually an indicator of a new build + self.data = self.data[self.data["TENURE"] != IGNORED_TENURES] + def clean_multi_glaze_proportion(self, ignore_step: bool = False) -> None: """ If there is no multi-glaze proportion but the windows are fully glazed, then we should assume a score of 100 diff --git a/etl/epc/settings.py b/etl/epc/settings.py index 87f27972..7100b0e9 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -215,6 +215,10 @@ EARLIEST_EPC_DATE = "2014-08-01" IGNORED_TRANSACTION_TYPES = "new dwelling" IGNORED_FLOOR_LEVELS = ["top floor", "mid floor"] IGNORED_PROPERTY_TYPES = "Park home" +IGNORED_TENURES = [ + "Not defined - use in the case of a new dwelling for which the intended tenure in not known. It is not to be used " + "for an existing dwelling" +] RDSAP_RESPONSE = "CURRENT_ENERGY_EFFICIENCY" HEAT_DEMAND_RESPONSE = "ENERGY_CONSUMPTION_CURRENT" diff --git a/etl/testing_data/retrofitted_properties.py b/etl/testing_data/retrofitted_properties.py index dcd3d09b..5e235c5f 100644 --- a/etl/testing_data/retrofitted_properties.py +++ b/etl/testing_data/retrofitted_properties.py @@ -18,6 +18,12 @@ def app(): test_file = pd.DataFrame( [ + # Live West Properties + {"address": "42, Foxes Field", "postcode": "TR18 3RJ", "Notes": None}, + {"address": "11, Cranley Gardens", "postcode": "TQ13 8UT", "Notes": None}, + # Keyzy properties + {'address': '2 South Terrace', 'postcode': 'NN1 5JY', 'Notes': ''}, + {'address': '25 Albert Street', 'postcode': 'PO12 4TY', 'Notes': ''}, # Pilot properties {'address': '113 Tenby Road', 'postcode': 'B13 9LT', 'Notes': ''}, {'address': '139 School Road', 'postcode': 'B28 8JF', 'Notes': ''}, diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index cd52bea9..788d1ad1 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -23,6 +23,34 @@ class LightingRecommendations: self.material = material[0] self.recommendation = [] + @staticmethod + def estimate_lighting_impact(number_of_bulbs: int): + """ + Placeholder function to estimate the actual energy savings of LEDs vs traditional lighting + :return: + """ + + wattage_incandescent = 60 # wattage of typical incandescent bulb in watts + wattage_led = 10 # wattage of typical LED bulb in watts + hours_per_day = 3 # average usage in hours per day + days_per_year = 365 # days in a year + national_grid_carbon_intensity = 162 # gCO2/kWh, average for 2023 in the UK + + # Energy usage per year for incandescent and LED bulbs (in kWh) + energy_usage_incandescent_per_year = (wattage_incandescent / 1000) * hours_per_day * days_per_year + energy_usage_led_per_year = (wattage_led / 1000) * hours_per_day * days_per_year + + # Energy savings per bulb per year + energy_savings_per_bulb_per_year = energy_usage_incandescent_per_year - energy_usage_led_per_year + + # Total energy savings for all bulbs + total_energy_savings_per_year = energy_savings_per_bulb_per_year * number_of_bulbs + + carbon_reduction_grams = total_energy_savings_per_year * national_grid_carbon_intensity + carbon_reduction_tonnes = carbon_reduction_grams / 1_000_000 # converting grams to tonnes + + return total_energy_savings_per_year, carbon_reduction_tonnes + def recommend(self): """ This method will check if there are any lighting fittings that aren't low energy. @@ -58,6 +86,8 @@ class LightingRecommendations: else: description = "Install low energy lighting in %s outlets" % str(number_non_lel_outlets) + heat_demand_change, carbon_change = self.estimate_lighting_impact(number_non_lel_outlets) + self.recommendation = [ { "parts": [], @@ -68,6 +98,8 @@ class LightingRecommendations: # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to # the proportion of lights that will be set to low energy "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2), + "heat_demand": heat_demand_change, + "co2_equivalent_savings": carbon_change, **cost_result } ] diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 2b35ffea..0444006d 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -143,6 +143,10 @@ class Recommendations: for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: + # We don't use the model for low energy lighting at the moment + if rec["type"] == "low_energy_lighting": + continue + new_heat_demand = property_heat_predictions[property_heat_predictions["recommendation_id"] == str( rec["recommendation_id"] )]["predictions"].values[0] @@ -151,12 +155,10 @@ class Recommendations: rec["recommendation_id"] )]["predictions"].values[0] - # We don't use the model for low energy lighting at the moment - if rec["type"] != "low_energy_lighting": - new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str( - rec["recommendation_id"] - )]["predictions"].values[0] - rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"]) + new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str( + rec["recommendation_id"] + )]["predictions"].values[0] + rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"]) if rec["type"] == "mechanical_ventilation": # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2