diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/Property.py b/backend/Property.py index c9cad22f..de87099b 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -192,15 +192,17 @@ class Property: recommendation_record["walls_insulation_thickness_ending"] = "none" # Update description to indicate it's insulate - if recommendation["type"] in ["solid_floor_insulation", "suspended_floor_insulation", - "exposed_floor_insulation"]: + if recommendation["type"] in [ + "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation" + ]: if len(recommendation["parts"]) > 1: raise NotImplementedError("Have more than 1 floor insulation part - handle this case") - recommendation_record["floor_thermal_transmittance_ending"] = recommendation["new_u_value"] + # recommendation_record["floor_thermal_transmittance_ending"] = recommendation["new_u_value"] # We don't really see above average for this in the training data recommendation_record["floor_insulation_thickness_ending"] = "average" - recommendation_record["floor_energy_eff_ending"] = "Good" + # This is rarely ever populated in the training data + # recommendation_record["floor_energy_eff_ending"] = "Good" else: if recommendation_record["floor_thermal_transmittance_ending"] is None: raise ValueError("We should not have a None value for the u value") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index b3d1c623..811c3c09 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -239,7 +239,6 @@ async def trigger_plan(body: PlanTriggerRequest): # This is a temporary step, to estimate the impact of the measured on heat demand and carbon # TODO: This needs to be cleaned up, if it happens to be kept - combined_recommendations_scoring_data = [] representative_recs = {} for property_id, property_recommendations in recommendations.items(): default_recommendations = [r for r in property_recommendations if r["default"]] @@ -276,58 +275,26 @@ async def trigger_plan(body: PlanTriggerRequest): representative_recs[property_id] = default_recommendations - property_instance = [p for p in input_properties if p.id == property_id][0] - - recommendation_record = property_instance.base_difference_record.df.to_dict("records")[0].copy() - - scoring_dict = {} - for rec in default_recommendations: - scoring_dict = Property.create_recommendation_scoring_data( - property_id=property_instance.id, - recommendation_record=recommendation_record, - recommendation=rec - ) - # At each iterations, we update the recommendation record with the changes reflectecd in the - # scoring_dict - for k in scoring_dict.keys(): - if k in recommendation_record.keys(): - recommendation_record[k] = scoring_dict[k] - - combined_recommendations_scoring_data.append(scoring_dict) - - # PERFORM SAME STEPS AGAIN - TODO: TO BE REMOVED - combined_recommendations_scoring_data = pd.DataFrame(combined_recommendations_scoring_data) - - all_combined_predictions = model_api.predict_all( - df=combined_recommendations_scoring_data, - bucket=get_settings().DATA_BUCKET, - prediction_buckets={ - "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, - "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, - "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET - } - ) - # We update the carbon and heat demand predictions + # TODO: The api call producing all_combined_predictions has been removed so we can potentially completely + # refactor this block to just perform the energy adjustments for property_id, property_recommendations in recommendations.items(): - combined_heat_demand = all_combined_predictions["heat_demand_predictions"] - combined_heat_demand = combined_heat_demand[combined_heat_demand["property_id"] == str(property_id)] - - combined_carbon = all_combined_predictions["carbon_change_predictions"] - combined_carbon = combined_carbon[combined_carbon["property_id"] == str(property_id)] property_instance = [p for p in input_properties if p.id == property_id][0] - carbon_change = float( - property_instance.data["co2-emissions-current"] - ) - combined_carbon["predictions"].values[0] + heat_demand_change = sum( + x.get("heat_demand", 0) for x in representative_recs[property_id] if + x["type"] not in ["mechanical_ventilation", "low_energy_lighting"] + ) + carbon_change = sum( + x.get("co2_equivalent_savings", 0) for x in representative_recs[property_id] if + x["type"] not in ["mechanical_ventilation", "low_energy_lighting"] + ) starting_heat_demand = ( float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area ) - expected_heat_demand = starting_heat_demand - ( - combined_heat_demand["predictions"].values[0] * property_instance.floor_area - ) + expected_heat_demand = starting_heat_demand - heat_demand_change # We don't want to adjust the heat demand for mechanical ventilation so we add it back on @@ -361,6 +328,21 @@ async def trigger_plan(body: PlanTriggerRequest): in representative_recs[property_id] ] representative_rec_data = pd.DataFrame(representative_rec_data) + # Suppress mechanical ventilation to have zero heat demand and co2 + representative_rec_data.loc[ + representative_rec_data["type"] == "mechanical_ventilation", "co2_equivalent_savings" + ] = 0 + representative_rec_data.loc[ + representative_rec_data["type"] == "mechanical_ventilation", "heat_demand" + ] = 0 + # Supress low energy lighting to have zero heat demand and co2 - this does not get affected by this process + representative_rec_data.loc[ + representative_rec_data["type"] == "low_energy_lighting", "co2_equivalent_savings" + ] = 0 + representative_rec_data.loc[ + representative_rec_data["type"] == "low_energy_lighting", "heat_demand" + ] = 0 + # Convert co2 and heat demand to proportions of their column sums representative_rec_data["co2_equivalent_savings_percent"] = ( representative_rec_data["co2_equivalent_savings"] / @@ -393,10 +375,18 @@ async def trigger_plan(body: PlanTriggerRequest): rec["co2_equivalent_savings"] = 0 rec["heat_demand"] = 0 rec["energy_cost_savings"] = 0 + elif rec["type"] == "low_energy_lighting": + # We do not convert, we just calculate energy cost savings + rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"]) + continue else: rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0] rec["heat_demand"] = change_data["heat_demand"].values[0] - rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"]) + # If the recommendation is solar, the savings are entirely in electricity + if rec["type"] == "solar_pv": + rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"]) + else: + rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"]) # Update recommendations recommendations[property_id] = property_recommendations diff --git a/backend/app/utils.py b/backend/app/utils.py index 6801da65..ba5509e1 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -75,8 +75,8 @@ def sap_to_epc(sap_points: int | float): :return: """ - if sap_points <= 0 or sap_points > 100: - raise ValueError("SAP points should be between 1 and 100.") + if sap_points <= 0: + raise ValueError("SAP points should be above 0.") if sap_points >= 92: return "A" diff --git a/backend/ml_models/AnnualBillSavings.py b/backend/ml_models/AnnualBillSavings.py index f3e5074a..99fae4db 100644 --- a/backend/ml_models/AnnualBillSavings.py +++ b/backend/ml_models/AnnualBillSavings.py @@ -29,6 +29,15 @@ class AnnualBillSavings: """ return cls.PRICE_FACTOR * kwh + @classmethod + def estimate_electric(cls, kwh: float): + """ + Estimate the annual bill savings based on the kwh savings + :param kwh: The kwh savings + :return: An estimate for annual bill savings + """ + return cls.ELECTRICITY_PRICE_CAP * kwh + @classmethod def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating): """ @@ -66,10 +75,10 @@ class AnnualBillSavings: # This should be negative consumption_difference = gradient * epc_energy_consumption + intercept - if consumption_difference > 0: - raise ValueError("consumption_difference should be negative") adjusted_consumption = (epc_energy_consumption + consumption_difference) + if adjusted_consumption < 0: + raise ValueError("consumption_difference should be negative") return adjusted_consumption diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 4615d2c4..5a210d9f 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -8,6 +8,7 @@ from etl.epc.settings import ( IGNORED_TRANSACTION_TYPES, IGNORED_FLOOR_LEVELS, IGNORED_PROPERTY_TYPES, + IGNORED_TENURES, FULLY_GLAZED_DESCRIPTIONS, AVERAGE_FIXED_FEATURES, BUILT_FORM_REMAP, @@ -632,6 +633,7 @@ class EPCDataProcessor: violation_missing_hotwater_description = pd.isnull(self.data["HOTWATER_DESCRIPTION"]) violation_missing_roof_description = pd.isnull(self.data["ROOF_DESCRIPTION"]) violation_invalid_property_type = self.data["PROPERTY_TYPE"] == IGNORED_PROPERTY_TYPES + violation_invalid_tenure = self.data["TENURE"].isin(IGNORED_TENURES) violation_df = pd.concat( [ @@ -644,6 +646,7 @@ class EPCDataProcessor: violation_missing_hotwater_description, violation_missing_roof_description, violation_invalid_property_type, + violation_invalid_tenure, ], axis=1, keys=[ "violation_uprn_missing", @@ -655,6 +658,7 @@ class EPCDataProcessor: "violation_missing_hotwater_description", "violation_missing_roof_description", "violation_invalid_property_type", + "violation_invalid_tenure" ] ) @@ -697,6 +701,9 @@ class EPCDataProcessor: # EPCs) we'll ignore them from the model self.data = self.data[self.data["PROPERTY_TYPE"] != IGNORED_PROPERTY_TYPES] + # We remove EPCs where the tenure is unknown, but is usually an indicator of a new build + self.data = self.data[self.data["TENURE"] != IGNORED_TENURES] + def clean_multi_glaze_proportion(self, ignore_step: bool = False) -> None: """ If there is no multi-glaze proportion but the windows are fully glazed, then we should assume a score of 100 diff --git a/etl/epc/settings.py b/etl/epc/settings.py index 87f27972..7100b0e9 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -215,6 +215,10 @@ EARLIEST_EPC_DATE = "2014-08-01" IGNORED_TRANSACTION_TYPES = "new dwelling" IGNORED_FLOOR_LEVELS = ["top floor", "mid floor"] IGNORED_PROPERTY_TYPES = "Park home" +IGNORED_TENURES = [ + "Not defined - use in the case of a new dwelling for which the intended tenure in not known. It is not to be used " + "for an existing dwelling" +] RDSAP_RESPONSE = "CURRENT_ENERGY_EFFICIENCY" HEAT_DEMAND_RESPONSE = "ENERGY_CONSUMPTION_CURRENT" diff --git a/etl/testing_data/retrofitted_properties.py b/etl/testing_data/retrofitted_properties.py new file mode 100644 index 00000000..5e235c5f --- /dev/null +++ b/etl/testing_data/retrofitted_properties.py @@ -0,0 +1,61 @@ +""" +This script will create an input csv for the recommendation engine and upload it to S3, which can be used for +testing +""" +import pandas as pd +from utils.s3 import save_csv_to_s3 + +USER_ID = 8 +PORTFOLIO_ID = 62 + + +def app(): + """ + This portfolio contains propertyies that we have demo'd in pilots, or properties that were provided to us + as proprties that are being treated under funding scehemes and we have pre/post EPRs for + :return: + """ + + test_file = pd.DataFrame( + [ + # Live West Properties + {"address": "42, Foxes Field", "postcode": "TR18 3RJ", "Notes": None}, + {"address": "11, Cranley Gardens", "postcode": "TQ13 8UT", "Notes": None}, + # Keyzy properties + {'address': '2 South Terrace', 'postcode': 'NN1 5JY', 'Notes': ''}, + {'address': '25 Albert Street', 'postcode': 'PO12 4TY', 'Notes': ''}, + # Pilot properties + {'address': '113 Tenby Road', 'postcode': 'B13 9LT', 'Notes': ''}, + {'address': '139 School Road', 'postcode': 'B28 8JF', 'Notes': ''}, + {'address': '77 Simmons Drive', 'postcode': 'B32 1SL', 'Notes': ''}, + {'address': 'Flat 2, 54 Wedgewood Road', 'postcode': 'B32 1LS', 'Notes': ''}, + # Warmfront ECO4 Properties + {'address': '73 Long Chaulden', 'postcode': 'HP1 2HX', 'Notes': ''}, + {'address': '8 Lindlings', 'postcode': 'HP1 2HA', 'Notes': ''}, + {'address': '44 Lindlings', 'postcode': 'HP1 2HE', 'Notes': ''}, + {'address': '46 Chaulden Terrace', 'postcode': 'HP1 2AN', 'Notes': ''}, + # Osmosis SHDF Properties + {'address': '4, Heather Shaw', 'postcode': 'BA14 7JS', 'Notes': ''}, + {'address': '16 Glastonbury Road', 'postcode': 'M32 9PE', 'Notes': ''}, + {'address': '31 Loddon Way', 'postcode': 'BA15 1HG', 'Notes': ''}, + {'address': '62 Pearmain Drive', 'postcode': 'NG3 3DJ', 'Notes': ''}, + ] + + ) + + # Store the data in s3 + filename = f"{USER_ID}/{PORTFOLIO_ID}/eco4_shdf_retrofits.csv" + save_csv_to_s3( + dataframe=test_file, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(PORTFOLIO_ID), + "housing_type": "Social", + "goal": "Increase EPC", + "goal_value": "A", + "trigger_file_path": filename + } + print(body) diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index cd52bea9..788d1ad1 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -23,6 +23,34 @@ class LightingRecommendations: self.material = material[0] self.recommendation = [] + @staticmethod + def estimate_lighting_impact(number_of_bulbs: int): + """ + Placeholder function to estimate the actual energy savings of LEDs vs traditional lighting + :return: + """ + + wattage_incandescent = 60 # wattage of typical incandescent bulb in watts + wattage_led = 10 # wattage of typical LED bulb in watts + hours_per_day = 3 # average usage in hours per day + days_per_year = 365 # days in a year + national_grid_carbon_intensity = 162 # gCO2/kWh, average for 2023 in the UK + + # Energy usage per year for incandescent and LED bulbs (in kWh) + energy_usage_incandescent_per_year = (wattage_incandescent / 1000) * hours_per_day * days_per_year + energy_usage_led_per_year = (wattage_led / 1000) * hours_per_day * days_per_year + + # Energy savings per bulb per year + energy_savings_per_bulb_per_year = energy_usage_incandescent_per_year - energy_usage_led_per_year + + # Total energy savings for all bulbs + total_energy_savings_per_year = energy_savings_per_bulb_per_year * number_of_bulbs + + carbon_reduction_grams = total_energy_savings_per_year * national_grid_carbon_intensity + carbon_reduction_tonnes = carbon_reduction_grams / 1_000_000 # converting grams to tonnes + + return total_energy_savings_per_year, carbon_reduction_tonnes + def recommend(self): """ This method will check if there are any lighting fittings that aren't low energy. @@ -58,6 +86,8 @@ class LightingRecommendations: else: description = "Install low energy lighting in %s outlets" % str(number_non_lel_outlets) + heat_demand_change, carbon_change = self.estimate_lighting_impact(number_non_lel_outlets) + self.recommendation = [ { "parts": [], @@ -68,6 +98,8 @@ class LightingRecommendations: # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to # the proportion of lights that will be set to low energy "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2), + "heat_demand": heat_demand_change, + "co2_equivalent_savings": carbon_change, **cost_result } ] diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 2b35ffea..0444006d 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -143,6 +143,10 @@ class Recommendations: for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: + # We don't use the model for low energy lighting at the moment + if rec["type"] == "low_energy_lighting": + continue + new_heat_demand = property_heat_predictions[property_heat_predictions["recommendation_id"] == str( rec["recommendation_id"] )]["predictions"].values[0] @@ -151,12 +155,10 @@ class Recommendations: rec["recommendation_id"] )]["predictions"].values[0] - # We don't use the model for low energy lighting at the moment - if rec["type"] != "low_energy_lighting": - new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str( - rec["recommendation_id"] - )]["predictions"].values[0] - rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"]) + new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str( + rec["recommendation_id"] + )]["predictions"].values[0] + rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"]) if rec["type"] == "mechanical_ventilation": # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2 diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 01cd4f17..8a773570 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -5,8 +5,8 @@ from recommendations.Costs import Costs class SolarPvRecommendations: # Approximate area of the solar panels SOLAR_PANEL_AREA = 1.6 - # Wattage per panel - SOLAR_PANEL_WATTAGE = 360 + # Wattage per panel - this is based on the average wattage of a solar panel being between 250w and 420w + SOLAR_PANEL_WATTAGE = 250 def __init__(self, property_instance): """ @@ -43,17 +43,20 @@ class SolarPvRecommendations: number_solar_panels = np.floor(self.property.solar_pv_roof_area / self.SOLAR_PANEL_AREA) solar_panel_wattage = number_solar_panels * self.SOLAR_PANEL_WATTAGE + roof_coverage_percent = round(self.property.solar_pv_percentage * 100) + # Given the wattage, we estimate the cost of the solar PV system. This is based on the MCS database # of solar PV installations cost_result = self.costs.solar_pv(wattage=solar_panel_wattage) - kw = int(np.round(solar_panel_wattage / 1000)) + kw = np.floor(solar_panel_wattage / 100) / 10 self.recommendation = [ { "parts": [], "type": "solar_pv", - "description": f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) panel system on the roof", + "description": f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) panel system on " + f"{roof_coverage_percent}% the roof", "starting_u_value": None, "new_u_value": None, "sap_points": None,