From ceb34979e4f82a4f77699c4592e36506d5bd4dfb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 12 Sep 2024 18:10:27 +0100 Subject: [PATCH 01/59] rough analysis for funding eligibility --- .../Cleethorpes Portfolio/epc data.py | 97 +++++++ etl/customers/bcc_tender/app.py | 10 +- etl/customers/newhaven/slides.py | 239 ++++++++++++++++++ etl/sfr/midlands_portfolio_est_funding.py | 159 ++++++++++++ 4 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 etl/customers/Cleethorpes Portfolio/epc data.py create mode 100644 etl/sfr/midlands_portfolio_est_funding.py diff --git a/etl/customers/Cleethorpes Portfolio/epc data.py b/etl/customers/Cleethorpes Portfolio/epc data.py new file mode 100644 index 00000000..a3ccbb2a --- /dev/null +++ b/etl/customers/Cleethorpes Portfolio/epc data.py @@ -0,0 +1,97 @@ +import os +import pandas as pd +from backend.SearchEpc import SearchEpc +from dotenv import load_dotenv +from tqdm import tqdm + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + + +def app(): + """ + Simple script to pull the EPC data for the Cleethorpes Portfolio + :return: + """ + + asset_list = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Updated Tenancy Schedule " + "Portfolio.xlsx", + ) + asset_list["row_id"] = asset_list.index + asset_list[" Street No."] = asset_list[" Street No."].astype(str) + + epc_data = [] + for _, property in tqdm(asset_list.iterrows(), total=len(asset_list)): + + if property[" Street No."] == "Ground Floor Commercial": + continue + uprn = property["Uprn"] + if not pd.isnull(uprn): + searcher = SearchEpc( + address1="", + postcode="", + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + uprn=int(uprn) + ) + searcher.find_property(skip_os=True) + else: + + if not pd.isnull(property[" Flat No."]) and property[" Flat No."] not in ["", " "]: + address1 = property[" Flat No."].strip() + ", " + property[" Street No."].strip() + else: + address1 = property[" Street No."].strip() + + if address1 == "1a Mews House 30": + address1 = "1a Rear of" + searcher = SearchEpc( + address1=address1, + postcode=property[" Postcode"].strip(), + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + uprn=None, + ) + searcher.get_epc() + # Get the newest record on lodgement-date + sorted_epcs = sorted( + searcher.data["rows"], key=lambda x: x["lodgement-date"] + ) + searcher.newest_epc = sorted_epcs[-1] + + if searcher.newest_epc is None: + raise ValueError(f"No EPC found for UPRN: {uprn}") + + epc_data.append( + { + "row_id": property["row_id"], + **searcher.newest_epc + } + ) + + epc_df = pd.DataFrame(epc_data) + + # Merge on data + asset_list_with_epc = asset_list.merge( + epc_df[["row_id", "address", "current-energy-rating", "current-energy-efficiency", "lodgement-date"]], + how="left", + left_on="row_id", + right_on="row_id", + ).rename( + columns={ + "address": "EPC Address", + "current-energy-rating": "Current EPC Rating", + "current-energy-efficiency": "Current SAP Score", + "lodgement-date": "EPC Date" + } + ) + + asset_list_with_epc.to_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Portfolio with EPCs.xlsx", + index=False + ) + + epc_df.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/epc_data.csv", + index=False + ) diff --git a/etl/customers/bcc_tender/app.py b/etl/customers/bcc_tender/app.py index 8cdc6e13..898db949 100644 --- a/etl/customers/bcc_tender/app.py +++ b/etl/customers/bcc_tender/app.py @@ -102,7 +102,7 @@ analysis_epcs = analysis_epcs[ [ "UPRN", "TENURE", "CURRENT_ENERGY_RATING", "WALLS_DESCRIPTION", "ROOF_DESCRIPTION", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA", "PROPERTY_TYPE", "BUILT_FORM", "MAINHEAT_DESCRIPTION", - "eligibility_type", + "eligibility_type", "PHOTO_SUPPLY", "ADDRESS1", "POSTCODE" ] ] analysis_epcs["grouped_epc_band"] = np.where( @@ -110,6 +110,14 @@ analysis_epcs["grouped_epc_band"] = np.where( "EPC D", "EPC E-G" ) + +analysis_epcs[pd.isnull(analysis_epcs["PHOTO_SUPPLY"])][["ADDRESS1", "POSTCODE"]].sample(1) + +analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].fillna(0) +analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].astype(float) +analysis_epcs["has_solar"] = np.where(analysis_epcs["PHOTO_SUPPLY"] > 0, 1, 0) +analysis_epcs["has_solar"].value_counts() + analysis_epcs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/bcc tender/analysis_epcs.csv", index=False) # Create aggregations and we store this information diff --git a/etl/customers/newhaven/slides.py b/etl/customers/newhaven/slides.py index 2fe914e2..61ed89cc 100644 --- a/etl/customers/newhaven/slides.py +++ b/etl/customers/newhaven/slides.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel +from utils.s3 import read_csv_from_s3 def get_data(portfolio_id, scenario_ids): @@ -415,3 +416,241 @@ def slides(): pd.set_option('display.max_rows', None) # Show more characters in a column pd.set_option('display.max_colwidth', None) + + # preparing of this data for the following 2 needs: + # 1) dataset to share with Nextgen heating + # 2) Breakdown of results by property type + + # get the asset list + asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath="8/90/pilot.csv") + asset_list = pd.DataFrame(asset_list) + # Get non-invasive recommendations + non_intrusive_recommendations = read_csv_from_s3( + bucket_name="retrofit-plan-inputs-dev", + filepath="8/90/non_invasive_recommendations.csv" + ) + non_intrusive_recommendations = pd.DataFrame(non_intrusive_recommendations) + + # Unnest this + import ast + survey_recs = [] + for _, row in non_intrusive_recommendations.iterrows(): + recs = ast.literal_eval(row["recommendations"]) + ashp_rec = next((r for r in recs if r["type"] == "air_source_heat_pump"), None) + solar_rec = next((r for r in recs if r["type"] == "solar_pv"), None) + to_append = { + "uprn": row["uprn"] + } + if ashp_rec["suitable"]: + to_append = { + **to_append, + "ashp_suitable": True, + "ashp_size_kw": ashp_rec["size"], + "ashp_cost": ashp_rec["cost"], + } + + if solar_rec["suitable"]: + to_append = { + **to_append, + "solar_suitable": True, + "solar_size_kwp": solar_rec["array_wattage"], + "solar_cost": solar_rec["cost"], + } + survey_recs.append(to_append) + survey_recs = pd.DataFrame(survey_recs) + + asset_list["uprn"] = asset_list["uprn"].astype(int) + survey_recs["uprn"] = survey_recs["uprn"].astype(int) + + vital_kwh = 7597 + domna_kwh = 10850 + scaling_factor = vital_kwh / domna_kwh + + next_gen_dataset = properties_df[[ + "uprn", "address", "postcode", + "property_type", "built_form", "current_energy_demand_heating_hotwater", + "mainfuel", "total_floor_area", "floor_height" + ]].rename( + columns={ + "mainfuel": "primary_fuel_type", + "total_floor_area": "gross_floor_area", + "current_energy_demand_heating_hotwater": "estimated_heating_hotwater_kwh" + } + ).merge( + asset_list[["uprn", "number_of_floors"]], + how="left", + on="uprn" + ).merge( + survey_recs, + how="left", + on="uprn" + ) + next_gen_dataset["estimated_heating_hotwater_kwh_scaled"] = ( + next_gen_dataset["estimated_heating_hotwater_kwh"] * scaling_factor + ) + + next_gen_dataset["ashp_suitable"] = next_gen_dataset["ashp_suitable"].fillna(False) + next_gen_dataset["solar_suitable"] = next_gen_dataset["solar_suitable"].fillna(False) + + # We prepare the scenario outputs by property type + grouped_data = next_gen_dataset.copy() + grouped_data["property_sub_type"] = grouped_data["built_form"].copy() + # If a property is a flat, re-map sub_type just to flat + grouped_data.loc[grouped_data["property_type"] == "Flat", "property_sub_type"] = "Flat" + # Same for maisonettes + grouped_data.loc[grouped_data["property_type"] == "Maisonette", "property_sub_type"] = "Maisonette" + + # We now pull out the recommendations impact by property type and sub type + + property_scenario_impact = [] + for scenario_id in scenario_ids: + # Get the recommendations for the scenario, default + scenario_recommendations = recommendations_df[ + (recommendations_df["Scenario ID"] == scenario_id) & + (recommendations_df["default"] == True) + ].copy() + + scenario_recommendations['ligting_kwh'] = scenario_recommendations.apply( + lambda x: x['kwh_savings'] if x['type'] == 'low_energy_lighting' else 0, + axis=1) + scenario_recommendations['solar_kwh'] = scenario_recommendations.apply( + lambda x: x['kwh_savings'] if x['type'] == 'solar_pv' else 0, axis=1) + + # Set 'Estimated Kwh Savings' to zero where specific kwh columns are used + scenario_recommendations['Estimated Kwh Savings'] = scenario_recommendations.apply( + lambda x: 0 if x['type'] in ['low_energy_lighting', 'solar_pv'] else x[ + 'kwh_savings'], axis=1) + + scenario_grouped_data = scenario_recommendations.groupby(['property_id']).agg({ + 'Estimated Kwh Savings': 'sum', + "estimated_cost": "sum" + }).reset_index() + + comparison = properties_df.drop_duplicates()[ + ["uprn", "property_id", "current_energy_demand_heating_hotwater"] + ].merge( + scenario_grouped_data, on=["property_id"], how="left" + ) + comparison["Estimated Kwh Savings"] = comparison["Estimated Kwh Savings"].fillna(0) + comparison["estimated_cost"] = comparison["estimated_cost"].fillna(0) + + comparison["post_scenario_heating_hotwater_kwh"] = ( + comparison["current_energy_demand_heating_hotwater"] - comparison["Estimated Kwh Savings"] + ) + comparison["scenario_id"] = scenario_id + + property_scenario_impact.append(comparison) + + property_scenario_impact = pd.concat(property_scenario_impact) + property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"]) + + # Scale + property_scenario_impact["post_scenario_heating_hotwater_kwh_scaled"] = ( + property_scenario_impact["post_scenario_heating_hotwater_kwh"] * scaling_factor + ) + + grouped_data = grouped_data.merge( + property_scenario_impact, how="left", on="uprn" + ) + + # Agg the data + grouped_data = grouped_data.groupby(["property_type", "property_sub_type", "scenario_id"]).agg({ + "estimated_heating_hotwater_kwh": "mean", + "estimated_heating_hotwater_kwh_scaled": "mean", + "estimated_cost": "mean", + "post_scenario_heating_hotwater_kwh": "mean", + "post_scenario_heating_hotwater_kwh_scaled": "mean" + }).reset_index() + + scenario_names = pd.DataFrame( + [ + { + "scenario_id": 47, + "scenario": "Demand Reduction – cavity & roof insulation", + }, + { + "scenario_id": 48, + "scenario": "Demand reduction – no solid wall, floors or heating/renewables", + }, + { + "scenario_id": 49, + "scenario": "Demand reduction – no decant" + }, + { + "scenario_id": 50, + "scenario": "Demand reduction – no decant + heating & solar", + }, + { + "scenario_id": 51, + "scenario": "Whole house retrofit" + } + ] + + ) + + grouped_data = grouped_data.merge( + scenario_names, how="left", on="scenario_id" + ) + + if not grouped_data[ + grouped_data["estimated_heating_hotwater_kwh"] < grouped_data["post_scenario_heating_hotwater_kwh"]].empty: + raise Exception("someting went wrong") + + if not grouped_data[grouped_data["estimated_heating_hotwater_kwh_scaled"] < grouped_data[ + "post_scenario_heating_hotwater_kwh_scaled"]].empty: + raise Exception("someting went wrong") + + # Reorder the columns + grouped_data = grouped_data[ + [ + 'property_type', + 'property_sub_type', + 'scenario', + 'estimated_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh', + 'estimated_heating_hotwater_kwh_scaled', + 'post_scenario_heating_hotwater_kwh_scaled', + 'estimated_cost', + ] + ] + + grouped_data = grouped_data.rename( + columns={ + "property_type": "Property Type", + "property_sub_type": "Property Sub Type", + "scenario": "Scenario", + "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh", + "post_scenario_heating_hotwater_kwh": "Post Scenario Heating & Hot Water kwh", + "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh (scaled)", + "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh (scaled)", + "estimated_cost": "Estimated Cost or Retrofit", + } + ) + + grouped_data.to_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Scenario kWh Impact by Property " + "Type.xlsx", + index=False + ) + + property_scenario_impact = property_scenario_impact.merge( + scenario_names, how="left", on="scenario_id" + ) + + df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario', + values=['post_scenario_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh_scaled']) + + # Flattening multi-index columns + df_pivot.columns = [f'{col[0]}_{col[1]}' for col in df_pivot.columns] + + # Reset the index to have a clean dataframe + df_pivot.reset_index(inplace=True) + + next_gen_dataset = next_gen_dataset.merge( + df_pivot, how="left", on="uprn" + ) + + next_gen_dataset.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/next_gen_dataset.csv", index=False + ) diff --git a/etl/sfr/midlands_portfolio_est_funding.py b/etl/sfr/midlands_portfolio_est_funding.py new file mode 100644 index 00000000..09102cfb --- /dev/null +++ b/etl/sfr/midlands_portfolio_est_funding.py @@ -0,0 +1,159 @@ +import msgpack + +import pandas as pd +from utils.s3 import read_from_s3 +from recommendations.recommendation_utils import ( + estimate_number_of_floors, esimtate_pitched_roof_area, estimate_external_wall_area, estimate_perimeter +) + + +def app(): + """ + Aims to estimate the amount of GBIS funding eligible + :return: + """ + + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + cleaned = msgpack.unpackb(cleaned, raw=False) + + epc_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Downloads/20240820 portfolio_epc_data.xlsx" + ) + + # For simplicity, get roofs or cavities + epc_data = epc_data.merge( + pd.DataFrame(cleaned["roof-description"]), + how="left", + left_on="ROOF_DESCRIPTION", + right_on="original_description" + ) + + epc_data["needs_roof_work"] = epc_data["insulation_thickness"].isin( + [ + None, + "100", + '150', + '50', + '75', + 'below average', + '25', + '12' + ] + ) & (epc_data["is_flat"] | epc_data["is_pitched"]) + + epc_data = epc_data.merge( + pd.DataFrame(cleaned["walls-description"]), + how="left", + left_on="WALLS_DESCRIPTION", + right_on="original_description", + suffixes=("", "_wall") + ) + + epc_data["needs_cavity_done"] = epc_data["is_cavity_wall"] & epc_data["insulation_thickness_wall"].isin( + ['none', "below average"] + ) + + loft_insulation_per_m2 = 16.07 + flat_roof_insulation_per_m2 = 195 + cwi_per_m2 = 14.21 + gbis_abs = 30 + + # We assume the work will take the home from a high D to a low D + def get_abs(floor_area): + if floor_area <= 72: + return 155 + + if floor_area <= 97: + return 169 + + if floor_area <= 199: + return 196.4 + + return 350.1 + + estimated_costs = [] + for _, home in epc_data.iterrows(): + to_append = { + "uprn": home["UPRN"], + "address": home["ADDRESS"], + "postcode": home["POSTCODE"], + } + + project_abs = get_abs(home["TOTAL_FLOOR_AREA"]) + available_funding = project_abs * gbis_abs + + n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"]) + floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5 + + # Check if it needs the walls done + if home["needs_cavity_done"]: + # We estimate the amount of insulation required + est_perimeter = estimate_perimeter( + floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, + num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors + ) + + insulation_needed = estimate_external_wall_area( + num_floors=n_floors, + floor_height=floor_height, + perimeter=est_perimeter, + built_form=home["BUILT_FORM"], + ) + cost_of_insulation = insulation_needed * cwi_per_m2 + + if available_funding > cost_of_insulation: + available_funding = cost_of_insulation + + to_append = { + **to_append, + "available_funding": available_funding, + "measure": "Cavity Wall Insulation", + "project_abs": project_abs + } + + estimated_costs.append(to_append) + continue + + if home["needs_roof_work"]: + # We estimate how much the cost of insulation would be + if home["is_pitched"]: + measure = "Loft Insulation" + + roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors + cost_of_insulation = roof_area * loft_insulation_per_m2 + else: + measure = "Flat Roof Insulation" + roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors + cost_of_insulation = roof_area * flat_roof_insulation_per_m2 + + if available_funding > cost_of_insulation: + available_funding = cost_of_insulation + + to_append = { + **to_append, + "available_funding": available_funding, + "measure": measure, + "project_abs": project_abs + } + + estimated_costs.append(to_append) + continue + + estimated_costs = pd.DataFrame(estimated_costs) + + estimated_costs.groupby("measure")["available_funding"].mean() + estimated_costs["measure"].value_counts() + + estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv") + + epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") + + n_properties_for_ashp = epc_data[ + (epc_data["PROPERTY_TYPE"] == "House") & + (epc_data["BUILT_FORM"].isin(["Detached", "Semi-Detached"])) + ].shape[0] From 15f55c021f694492a925204f54ca975bd52b0702 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 13 Sep 2024 15:31:43 +0100 Subject: [PATCH 02/59] AIHA data review WIP --- backend/SearchEpc.py | 92 ++++++- etl/customers/aiha/epc_data_pull.py | 363 ++++++++++++++++++++++++++++ 2 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 etl/customers/aiha/epc_data_pull.py diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index 5f101d81..b5ec8c46 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -7,6 +7,9 @@ import pandas as pd import numpy as np from epc_api.client import EpcClient from backend.OrdnanceSurvey import OrdnanceSuveyClient +from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes +from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes +from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from BaseUtility import Definitions from utils.logger import setup_logger from typing import List @@ -181,6 +184,7 @@ class SearchEpc: self.newest_epc = None self.older_epcs = None self.full_sap_epc = None + self.metadata = None # These are the address and postcode values, which we store in the database self.address_clean = None @@ -306,7 +310,10 @@ class SearchEpc: if (property_type is None) and (address is None): return rows - if len(uprns) == 1: + unique_property_types = {r["property-type"] for r in rows} + + # We allow for variation in property type across flats/maisonettes + if (len(uprns) == 1) and ((len(unique_property_types) == 1) or unique_property_types == {"Flat", "Maisonette"}): return rows if property_type is not None: @@ -784,3 +791,86 @@ class SearchEpc: self.address_clean = self.ordnance_survey_client.address_os self.postcode_clean = self.ordnance_survey_client.postcode_os return + + def check_attribute_variations(self): + attribute_map = { + "walls-description": { + "cleaner": WallAttributes, + "attribute": [ + "is_cavity_wall", "is_solid_brick", "is_system_built", "is_timber_frame", + "is_granite_or_whinstone", "is_cob", "is_sandstone_or_limestone", "is_park_home" + ], + "name": "has_wall_type_ever_varied" + }, + "roof-description": { + "cleaner": RoofAttributes, + "attribute": [ + "is_flat", "is_pitched", "is_roof_room", "is_thatched", "has_dwelling_above" + ], + "name": "has_roof_type_ever_varied" + }, + "floor-description": { + "cleaner": FloorAttributes, + "attribute": [ + "is_to_unheated_space", "is_to_external_air", "is_suspended", "is_solid", "is_to_external_air", + ], + "name": "has_floor_type_ever_varied" + } + } + + attribute_variations = {} + for attribute, attribute_objs in attribute_map.items(): + attribute_variations[attribute_objs["name"]] = False + cleaner = attribute_objs["cleaner"] + type_timeline = pd.DataFrame([cleaner(epc[attribute]).process() for epc in self.older_epcs] + [ + cleaner(self.newest_epc[attribute]).process() + ]) + # For eac col in attribute_objs["attribute"] we check if the timeline has ever varied, i.e has gone + # from true to false + for col in attribute_objs["attribute"]: + if type_timeline[col].nunique() > 1: + attribute_variations[attribute_objs["name"]] = True + break + + return attribute_variations + + def identify_flat_floor(self): + # If there is no dwelling above, it is a top floor flat + processed_roof = RoofAttributes(self.newest_epc["roof-description"]).process() + if not processed_roof["has_dwelling_above"]: + return "top" + + # We know that there is a dwelling above. If there's also a drwelling below, it is a mid floor flat + processed_floor = FloorAttributes(self.newest_epc["floor-description"]).process() + if processed_floor["another_property_below"]: + return "mid" + + # Otherwise ground floor + return "ground" + + def get_metadata(self): + if self.newest_epc is None: + raise ValueError("No EPC data available") + + # We check if the property has ever been downgraded on SAP + has_sap_ever_downgraded = False + sap_timeline = [int(epc["current-energy-efficiency"]) for epc in self.older_epcs] + [ + int(self.newest_epc["current-energy-efficiency"]) + ] + # We check if there has ever been a decrease by differencing + has_sap_ever_downgraded = any(np.diff(sap_timeline) < 0) + + # We check if the wall type has ever varied over time + attribute_varations = self.check_attribute_variations() + + # If the property is a flat, we distinguish between top, mid, ground floor + floor = None + if self.newest_epc["property-type"] == "Flat": + floor = self.identify_flat_floor() + + self.metadata = { + "days_since_last_epc": (pd.Timestamp.now() - pd.Timestamp(self.newest_epc["lodgement-date"])).days, + "has_sap_ever_downgraded": has_sap_ever_downgraded, + "floor": floor, + **attribute_varations + } diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py new file mode 100644 index 00000000..8aaaf5ba --- /dev/null +++ b/etl/customers/aiha/epc_data_pull.py @@ -0,0 +1,363 @@ +import os +from tqdm import tqdm +from dotenv import load_dotenv +import pandas as pd +from backend.SearchEpc import SearchEpc +from etl.spatial.OpenUprnClient import OpenUprnClient + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + +pd.set_option('display.max_rows', 500) +pd.set_option('display.max_columns', 500) +pd.set_option('display.width', 1000) + + +def app(): + # Retrieve EPC data for the SHDF AIHA portfolio + + data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Khalim Review - 240902 - KSQ - AIHA - SHDF Wave " + "3 bid - Supplementary information.xlsx", + sheet_name="All units information", + header=3 + ) + + # Remove the .eg row + data = data.tail(-1) + + # Remove the bottom 2 rows + data = data.head(-2) + data = data.reset_index(drop=True) + data["row_id"] = data.index + + ammendments = { + "12 11-18 Schonfeld Square": "12 Schonfeld Square", + "35 35-37 Schonfeld Square": "35 Schonfeld Square", + '77 Schonfeld Square': '77 Lordship Road', + "83 Lordship Road (Schonfeld Square)": "83 Lordship Road", + "A 80 Bethune Road": "80A Bethune Road", + "86B Bethune Road": "Flat B, 86 Bethune Road", + "22 Glendale Road": "22 Glendale Avenue", + "121 Southbourne Road": "121 Southbourne Grove", + } + + no_epc = [ + "80B Bethune Road", + "89B Manor Road", + "12 Monkville Avenue", + "9 Greenview", + ] + + property_type_map = { + "House, mid-terrace": "House", + "House, end terrace": "House", + "House, semi-detached": "House", + "House, detached": "House", + "Flat": "Flat", + } + + epc_data = [] + epc_metadata = [] + for _, home in tqdm(data.iterrows(), total=len(data)): + + # Build address 1 based on if there is: + # 1) Address letter or number + # 2) Street address + + modified = False + address1 = "" + address1_backup = "" + + if home["Address letter or number"] in ["A", "B", "C"]: + + house_no = home['Street address'].split(' ')[0] + street = ' '.join(home['Street address'].split(' ')[1:]) + address1 = f"{house_no}{home['Address letter or number']} {street}" + + address1_backup = f"Flat {home['Address letter or number']} {house_no} {street}" + modified = True + + else: + if not pd.isnull(home["Address letter or number"]): + address1 += f"{home['Address letter or number']} " + if not pd.isnull(home["Street address"]): + address1 += f"{home['Street address']}" + address1 = address1.strip() + + if address1.split(" ")[-1].lower() == "rd": + # Replace with road + address1 = address1.lower().replace(" rd", " road") + + # Specific ammendments + if address1 in ammendments: + address1 = ammendments[address1] + + if address1 in no_epc: + continue + + searcher = SearchEpc( + address1=address1, + postcode=home["Postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=property_type_map[home["Property type"]] + ) + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None and modified: + searcher = SearchEpc( + address1=address1_backup, + postcode=home["Postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=property_type_map[home["Property type"]] + ) + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None: + raise Exception("Not found") + + epc_data.append( + { + "row_id": home["row_id"], + **searcher.newest_epc + } + ) + + searcher.get_metadata() + + epc_metadata.append( + { + "row_id": home["row_id"], + "address": address1, + "postcode": home["Postcode"], + **searcher.metadata + } + ) + + epc_metadata = pd.DataFrame(epc_metadata) + epc_data = pd.DataFrame(epc_data) + + # Check matched addresses + matched_addresses = epc_metadata[["row_id", "address", "postcode"]].copy() + matched_addresses = matched_addresses.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + + # We look for differences between the asset list and the EPC data + comparison_cols = { + "Property type": [ + { + "epc_col": "property-type", + "map": property_type_map + }, + { + "epc_col": "built-form", + "map": { + "House, mid-terrace": "Mid-Terrace", + "House, end terrace": "End-Terrace", + "House, semi-detached": "Semi-Detached", + "House, detached": "Detached", + "Flat": "Flat", + } + } + ], + "Energy starting band (EPC)": [ + { + "epc_col": "current-energy-rating", + "map": {} + } + ], + "Wall type": [ + { + "epc_col": "walls-description", + "search_terms": { + "solid": "Solid brick", + "cavity": "Cavity wall", + "solid - internal lining": "Solid brick", + } + } + ], + "Roof type": [ + { + "epc_col": "roof-description", + "search_terms": { + "pitched": "Pitched", + "n/a - (flat above)": "another dwelling above" + } + } + ], + "Floor type": [ + { + "epc_col": "floor-description", + "search_terms": { + "solid": "Solid", + "suspended": "Suspended", + "solid - floating floor for services": "Solid" + } + } + ], + } + + import re + differences = [] + for asset_list_col, list_of_configs in comparison_cols.items(): + + if asset_list_col in ["Wall type", "Roof type", "Floor type"]: + config = list_of_configs[0] + # We handle this differently + remapped = data[["row_id", asset_list_col]].copy() + # Strip the asset list col incase of leading/trailing spaces + remapped[asset_list_col] = remapped[asset_list_col].str.strip() + remapped[asset_list_col] = remapped[asset_list_col].str.lower() + remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner") + # We do a search term check + remapped["Match"] = None + for search_term, epc_term in config["search_terms"].items(): + if "/" in search_term: + escaped_search_term = re.escape(search_term) + remapped.loc[remapped[asset_list_col].str.contains(escaped_search_term), "Match"] = ( + remapped.loc[ + remapped[asset_list_col].str.contains(escaped_search_term), config["epc_col"] + ].str.contains(epc_term) + ) + else: + remapped.loc[remapped[asset_list_col].str.contains(search_term), "Match"] = ( + remapped.loc[ + remapped[asset_list_col].str.contains(search_term), config["epc_col"] + ].str.contains(epc_term) + ) + + if pd.isnull(remapped["Match"]).sum(): + raise Exception("Not all matched") + + remapped["Match"] = remapped["Match"].astype(bool) + + if not all(remapped["Match"]): + differences.append( + { + "Column": asset_list_col, + "Differences": remapped[~remapped["Match"]], + } + ) + + continue + + for config in list_of_configs: + + remapped = data[["row_id", asset_list_col]].copy() + if config["map"]: + remapped[asset_list_col] = remapped[asset_list_col].map(config["map"]) + + # Merge on + remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner") + remapped["Match"] = remapped[asset_list_col] == remapped[config["epc_col"]] + if not all(remapped["Match"]): + differences.append( + { + "Column": asset_list_col, + "Differences": remapped[~remapped["Match"]], + } + ) + + # Check for property type + property_type_differences = differences[0]["Differences"].copy() + property_type_differences = property_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + print(property_type_differences) + + # Check for built form + built_form_differences = differences[1]["Differences"].copy() + built_form_differences = built_form_differences[built_form_differences["Property type"] != "Flat"] + built_form_differences = built_form_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + print(built_form_differences) + + # Check for energy rating + energy_rating_differences = differences[2]["Differences"].copy() + energy_rating_differences = energy_rating_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(energy_rating_differences) + + # Check for wall type + wall_type_differences = differences[3]["Differences"].copy() + wall_type_differences = wall_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(wall_type_differences) # Many wall type differences + + # Check for roof type + roof_type_differences = differences[4]["Differences"].copy() + roof_type_differences = roof_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(roof_type_differences) # Many roof type differences + + # Check for floor type + floor_type_differences = differences[5]["Differences"].copy() + floor_type_differences = floor_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(floor_type_differences) # Many floor type differences + + # TODO: 47 Ashtead Road [100021024699] shows solid brick wall on EPC - is probably cavity wall + + # We have the EPC data. Let's check conservation area/historic/listed building status + portfolio_spatial_data = OpenUprnClient.get_spatial_data( + epc_data["uprn"].unique().tolist(), bucket_name="retrofit-data-dev" + ) + + portfolio_spatial_data["UPRN"] = portfolio_spatial_data["UPRN"].astype(str) + + spatial_data = data[["row_id", "Planning constraints"]].merge( + epc_data[["row_id", "uprn"]], on="row_id", how="left", + + ).merge( + portfolio_spatial_data[["UPRN", "conservation_status", "is_listed_building", "is_heritage_building"]], + left_on="uprn", + right_on="UPRN", how="left" + ) + + spatial_data[ + (spatial_data["Planning constraints"] == "None") + ]["conservation_status"].value_counts() + + # One property is in a conservation area, that was not picked up in the asset data + print(spatial_data[ + (spatial_data["Planning constraints"] == "None") & + (spatial_data["conservation_status"] == True) + ].merge( + data[["row_id", "Address letter or number", "Street address", "Postcode"]], on="row_id", how="left" + )) + + # All properties match up apart from one where the asset data indicates it's in a conservation area, however + # the sparital data indicates it's not. There do not appear to be any listed/heritage buildings in the portfolio + + # Draft archetyping + archetyping_data = data[ + [ + "row_id", + "Energy starting band (EPC)", + "Property type", + "Property year built", + "Gross internal area (sqm)", + "Current heating system type", + "Wall type", + "Floor type", + "Roof type", + "Window type", + "Location (Floor)", + ] + ] From 391c6f5cf0e07eca36e8d2ecf8c075e475df95b5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 13 Sep 2024 18:05:02 +0100 Subject: [PATCH 03/59] Adding to archetyping --- etl/customers/aiha/epc_data_pull.py | 408 ++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 8aaaf5ba..5e7c6714 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -2,6 +2,9 @@ import os from tqdm import tqdm from dotenv import load_dotenv import pandas as pd +import numpy as np +import msgpack +from utils.s3 import read_from_s3 from backend.SearchEpc import SearchEpc from etl.spatial.OpenUprnClient import OpenUprnClient @@ -345,7 +348,63 @@ def app(): # All properties match up apart from one where the asset data indicates it's in a conservation area, however # the sparital data indicates it's not. There do not appear to be any listed/heritage buildings in the portfolio + ################################################################ # Draft archetyping + ################################################################ + + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + cleaned = msgpack.unpackb(cleaned, raw=False) + + epc_data = epc_data.merge( + pd.DataFrame(cleaned["walls-description"])[ + ['original_description', + 'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick', 'is_system_built', 'is_timber_frame', + 'is_as_built', 'is_assumed', 'insulation_thickness'] + + ].rename( + columns={ + "is_solid_brick": "is_solid_brick_wall", + "is_system_built": "is_system_built_wall", + "is_timber_frame": "is_timber_frame_wall", + "is_assumed": "is_assumed_wall", + "insulation_thickness": "insulation_thickness_wall" + } + ), + left_on="walls-description", + right_on="original_description" + ).merge( + pd.DataFrame(cleaned["roof-description"])[ + [ + 'original_description', 'is_pitched', 'is_roof_room', 'is_loft', + 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed', + 'has_dwelling_above', 'insulation_thickness' + ] + ].rename( + columns={ + "is_assumed": "is_assumed_roof", + } + ), + left_on="roof-description", + right_on="original_description" + ).merge( + pd.DataFrame(cleaned["floor-description"])[ + [ + 'original_description', 'is_solid', 'is_suspended', 'is_assumed', + 'insulation_thickness' + ] + ].rename( + columns={ + "is_assumed": "is_assumed_floor", + "insulation_thickness": "insulation_thickness_floor" + } + ), + left_on="floor-description", + right_on="original_description" + ) + archetyping_data = data[ [ "row_id", @@ -360,4 +419,353 @@ def app(): "Window type", "Location (Floor)", ] + ].merge( + epc_metadata[["row_id", "floor"]], + how="left", + on="row_id" + ).merge( + epc_data[ + [ + "row_id", "uprn", "current-energy-rating", "property-type", "built-form", "total-floor-area", + 'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick_wall', 'is_system_built_wall', + 'is_timber_frame_wall', 'is_as_built', 'is_assumed_wall', 'insulation_thickness_wall', + 'is_solid', 'is_suspended', 'is_assumed_floor', 'insulation_thickness_floor', + 'is_pitched', 'is_roof_room', 'is_loft', + 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed_roof', + 'has_dwelling_above', 'insulation_thickness', "mainheat-description", + "local-authority-label" + ] + ], + how="left", + on="row_id" + ).merge( + spatial_data[["row_id", "conservation_status", ]], + on="row_id", + how="left" + ) + + if archetyping_data.shape[0] != data.shape[0]: + raise Exception("Mismatch in data") + + # We create groups analogous to the Energy Company Obligation + # 0 - 72, 73 - 97, 98 - 199, 200+ + archetyping_data["Floor_area_category"] = pd.cut( + archetyping_data["Gross internal area (sqm)"], + bins=[0, 72, 97, 199, 1000], + labels=["0-72", "73-97", "98-199", "200+"] + ) + archetyping_data["Floor_area_category_backup"] = pd.cut( + archetyping_data["total-floor-area"].astype(float), + bins=[0, 72, 97, 199, 1000], + labels=["0-72", "73-97", "98-199", "200+"] + ) + archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].fillna( + archetyping_data["Floor_area_category_backup"] + ) + archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].astype(str) + archetyping_data["Floor_area_category"] = np.where( + pd.isnull(archetyping_data["Floor_area_category"]), + "Unknown", + archetyping_data["Floor_area_category"] + ) + archetyping_data = archetyping_data.drop(columns=["Floor_area_category_backup"]) + + archetyping_data["property-type-reduced"] = np.where( + archetyping_data["property-type"].isin(["Flat", "Maisionette"]), + "Flat/Maisonette", + archetyping_data["property-type"] + ) + + archetyping_data["built-form-reduced"] = np.where( + archetyping_data["built-form"].isin(["End-Terrace", "Semi-Detached"]), + "End-Terrace/Semi-Detached", + archetyping_data["built-form"] + ) + archetyping_data["built-form-reduced"] = np.where( + archetyping_data["property-type-reduced"] == "Flat/Maisonette", + "Flat/Maisonette", + archetyping_data["built-form-reduced"] + ) + + archetyping_data["Wall type"] = np.where( + archetyping_data["Wall type"].isin(['Solid ', 'Solid - internal lining ']), + "Solid", + archetyping_data["Wall type"] + ) + archetyping_data["Wall type"] = np.where( + archetyping_data["Wall type"].isin(['Cavity ', 'cavity ']), + "Cavity", + archetyping_data["Wall type"] + ) + + # Proposed remaps based on discoveries + value_remaps = { + # 8 Filey Avenue + "100021040744": { + "variable": "Property type", + "newvalue": "House, mid-terrace", + }, + # 7 Yetev Lev Court + "100021032043": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 14 Yetev Lev Court + "100021032050": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 23 Yetev Lev Court + "100021032059": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 30 Yetev Lev Court + "100021032066": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 34 Yetev Lev Court + "100021032070": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # B 86 Bethune Road + "100021026285": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # A 80 Bethune Road + "100021026277": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # 140 Kyverdale Road + "100021052262": { + "variable": "Property type", + "newvalue": "House, mid-terrace", + }, + # 6 Leabourne Road + "100021053799": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # 22 Britannia Gardens - needs confirmation + # 7 Satanita Road - needs confirmation + # 12 Cheltenham Crescent + "100011402969": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + "100021031752": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + # 79 Craven Park Road + "100021169682": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + # 88 Darenth Road + "100021036148": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021036165": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021036167": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021053849": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021054353": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021054560": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021059839": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021059848": { + "variable": "Roof type", + "newvalue": "Room Roof" + } + } + + # Perform the remaps + for uprn, config in value_remaps.items(): + archetyping_data[config["variable"]] = np.where( + archetyping_data["uprn"].astype(str) == uprn, config["newvalue"], archetyping_data[config["variable"]] + ) + + # row_id = data[ + # # (data["Address letter or number"] == "C") & + # (data["Street address"].str.strip() == "41 Moresby Road") + # ]["row_id"] + # if len(row_id) != 1: + # raise Exception("Fail") + # print(epc_data[epc_data["row_id"] == row_id.values[0]]["uprn"]) + + # Map the year to the age band + def categorize_year(year): + if isinstance(year, str): + # Handle the case where year is in the format '1930s' + if 's' in year: + year = int(year[:4]) + else: + year = int(year) + else: + year = int(year) + + # Categorize based on year ranges + if year < 1900: + return 'A' + elif 1900 <= year <= 1929: + return 'B' + elif 1930 <= year <= 1949: + return 'C' + elif 1950 <= year <= 1966: + return 'D' + elif 1967 <= year <= 1975: + return 'E' + elif 1976 <= year <= 1982: + return 'F' + elif 1983 <= year <= 1990: + return 'G' + elif 1991 <= year <= 1995: + return 'H' + elif 1996 <= year <= 2002: + return 'I' + elif 2003 <= year <= 2006: + return 'J' + elif 2007 <= year <= 2011: + return 'K' + else: # year >= 2012 + return 'L' + + archetyping_data["SAP_age_band"] = archetyping_data["Property year built"].apply( + categorize_year + ) + + # Flag if the property is in London/Manchester + archetyping_data["Location"] = np.where( + archetyping_data["local-authority-label"].isin( + ["Hackney", "Barnet", "Haringey"] + ), + "London", + np.where( + archetyping_data["local-authority-label"].isin( + ["Salford", "Bury"] + ), + "Manchester", + "Southend" + ) + ) + # 9 Greenview is in manchester + archetyping_data["Location"] = np.where( + archetyping_data["row_id"] == data[data["Street address"] == "9 Greenview"]["row_id"].values[0], + "Manchester", + archetyping_data["Location"] + ) + + # Hackney 73 - London + # Southend-on-Sea 6 - Southend + # Barnet 4 - London + # Castle Point 4 - Southend + # Haringey 3 - London + # Salford 2 - Manchester + # Bury 1 - Manchester + + primary_archetyping_cols = [ + 'Property type', + "Location (Floor)", + 'Current heating system type', + 'Wall type', + 'Roof type', + "Location", + # 'current-energy-rating', 'property-type-reduced', 'built-form-reduced', 'is_cavity_wall', + # 'is_solid_brick_wall', 'is_system_built_wall', 'is_timber_frame_wall', 'is_as_built', + # 'is_solid', 'is_roof_room', + # 'is_loft', 'is_flat', 'is_thatched', + # 'is_at_rafters', 'has_dwelling_above', + # 'conservation_status', ] + + secondary_cols = [ + 'SAP_age_band', + 'is_filled_cavity', + 'insulation_thickness_wall' + 'insulation_thickness_floor' + 'insulation_thickness', + 'is_assumed_wall', + 'is_assumed_roof', + 'Floor_area_category' + ] + + archetypes = archetyping_data[primary_archetyping_cols].drop_duplicates() + # Hash the variables + archetypes["archetype_hash"] = archetypes.apply( + lambda x: hash(tuple(x.values)), + axis=1 + ) + archetypes = archetypes.sort_values("archetype_hash", ascending=True) + archetypes = archetypes.reset_index(drop=True) + archetypes["archetype_id"] = archetypes.index + + archetypes.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/basic-archetypes.csv", index=False) + + # We match properties to archetypes + archetyping_data = archetyping_data.merge( + archetypes, + on=primary_archetyping_cols, + how="left" + ) + + # We should choose a representative property for each archetype + archetyping_data = archetyping_data.merge( + epc_metadata[["row_id", "days_since_last_epc"]], + how="left", + on="row_id" + ) + + # Mark the property with the oldest EPC as the representative property + representative_properties = archetyping_data.sort_values( + ["archetype_id", "days_since_last_epc"], ascending=[True, False] + ).drop_duplicates("archetype_id") + + archetyping_data["for_sample"] = np.where( + archetyping_data["row_id"].isin(representative_properties["row_id"]), + True, + False + ) + + # We save the archetyping data + archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False) + # Save the EPC data + epc_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv", index=False) + # Save the spatial data + spatial_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge( + spatial_data, + on="row_id", + how="left" + ) + spatial_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/spatial_data.csv", index=False) + + # Save archetyping data + archetyping_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge( + archetyping_data, + on="row_id", + how="left" + ) + + archetyping_data = archetyping_data.drop(columns=["row_id"]) From 9b08d49b85fceca164e70e9eaf8c32f1c7442e78 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 15 Sep 2024 13:58:59 +0100 Subject: [PATCH 04/59] Adding another heating test --- etl/customers/aiha/epc_data_pull.py | 4 +- recommendations/HeatingRecommender.py | 19 ++++--- .../test_data/heating_recommendations_data.py | 54 +++++++++++++++++++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 5e7c6714..16081205 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -767,5 +767,5 @@ def app(): on="row_id", how="left" ) - - archetyping_data = archetyping_data.drop(columns=["row_id"]) + archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index dc433806..103fa7b1 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -9,6 +9,11 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: + ASSUMED_ELECTRIC_HEATING = [ + "Portable electric heaters assumed for most rooms", + "No system present, electric heaters assumed" + ] + ELECTRIC_HEATING_DESCRIPTIONS = [ "Room heaters, electric", "Electric storage heaters", @@ -16,6 +21,10 @@ class HeatingRecommender: "Portable electric heaters assumed for most rooms", ] + ROOM_HEATERS_DESCRIPTIONS = [ + "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", + ] + high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" def __init__(self, property_instance: Property): @@ -25,8 +34,8 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = ( - self.property.main_heating["clean_description"] in self.ELECTRIC_HEATING_DESCRIPTIONS + self.has_electric_heating_description = self.property.main_heating["clean_description"] in ( + self.ELECTRIC_HEATING_DESCRIPTIONS + self.ASSUMED_ELECTRIC_HEATING ) def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): @@ -37,9 +46,7 @@ class HeatingRecommender: # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we # can consider hhr storage heaters - electric_heating_assumed = ( - self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"] - ) + electric_heating_assumed = self.property.main_heating["clean_description"] in self.ASSUMED_ELECTRIC_HEATING has_electric = self.has_electric_heating_description or electric_heating_assumed @@ -64,7 +71,7 @@ class HeatingRecommender: # The property is using portable heaters and has access to gas mains has_room_heaters = ( - self.property.main_heating["clean_description"] in ["Room heaters, mains gas", "Room heaters, electric"] and + self.property.main_heating["clean_description"] in self.ROOM_HEATERS_DESCRIPTIONS and self.property.data["mains-gas-flag"] ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index f283050b..c64aab6f 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -348,6 +348,50 @@ testing_examples = [ "Programmer and room thermostat for heating controls so we'd expect an ASHP heating recommendation" "as the only option, and heating controls recommendations for programmer, room thermostats and trvs" "as well as ttzc" + }, + { + "epc": { + 'lmk-key': '977006769242013072314202915172178', 'address1': '40, Elswick Road', 'address2': None, + 'address3': None, 'postcode': 'B44 0JQ', 'building-reference-number': 7277661178, + 'current-energy-rating': 'G', 'potential-energy-rating': 'B', 'current-energy-efficiency': 18, + 'potential-energy-efficiency': 88, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2013-07-23', 'local-authority': 'E08000025', 'constituency': 'E14000561', + 'county': None, + 'lodgement-date': '2013-07-23', 'transaction-type': 'none of the above', 'environment-impact-current': 31, + 'environment-impact-potential': 90, 'energy-consumption-current': 536, 'energy-consumption-potential': 59, + 'co2-emissions-current': 7.0, 'co2-emiss-curr-per-floor-area': 96, 'co2-emissions-potential': 0.9, + 'lighting-cost-current': 48, 'lighting-cost-potential': 48, 'heating-cost-current': 1395, + 'heating-cost-potential': 353, 'hot-water-cost-current': 457, 'hot-water-cost-potential': 69, + 'total-floor-area': 73.0, 'energy-tariff': 'Unknown', 'mains-gas-flag': 'Y', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 4, 'number-heated-rooms': 1, + 'low-energy-lighting': 90, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, 75 mm loft insulation', + 'roof-energy-eff': 'Average', 'roof-env-eff': 'Average', + 'mainheat-description': 'Portable electric heaters assumed for most rooms', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 90% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', + 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': 0.0, + 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '40, Elswick Road', 'local-authority-label': 'Birmingham', + 'constituency-label': 'Birmingham, Erdington', 'posttown': 'BIRMINGHAM', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2013-07-23 14:20:29', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 9.0, 'uprn': 100070358594, + 'uprn-source': 'Address Matched' + }, + "heating_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [], + "notes": "" } ] @@ -389,3 +433,13 @@ print(eg["mainheat-energy-eff"]) print(eg["property-type"]) print(eg["built-form"]) print(eg["mainheatcont-description"]) + +### We also use the Midlands EPC F/G portfolio to get examples to create tests +portfolio = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" +) +portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +eg = portfolio[ + (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms") +].sample(1) +eg = eg.squeeze().to_dict() From c4ab7f5a2c6599f10290d20e1b8dc6f279f375ac Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 13:11:08 +0100 Subject: [PATCH 05/59] Adding new heating recommendation test --- etl/customers/aiha/epc_data_pull.py | 6 ++-- recommendations/HeatingControlRecommender.py | 4 +-- recommendations/HeatingRecommender.py | 26 ++++++++++----- .../test_data/heating_recommendations_data.py | 32 +++++++++++++++---- .../tests/test_heating_recommendations.py | 7 ---- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 16081205..f7f4631c 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -767,5 +767,7 @@ def app(): on="row_id", how="left" ) - archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", - index=False) + archetyping_data.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False + ) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 3e47c355..6f848441 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -121,7 +121,7 @@ class HeatingControlRecommender: self.recommendation.append( { - "description": "upgrade heating controls to High Heat Retention Storage Heater Controls", + "description": "Upgrade heating controls to High Heat Retention Storage Heater Controls", **self.costs.celect_type_controls(), "simulation_config": simulation_config, "description_simulation": description_simulation @@ -192,7 +192,7 @@ class HeatingControlRecommender: has_trvs=has_trvs ) - description = "upgrade heating controls to Room thermostat, programmer and TRVs" + description = "Upgrade heating controls to Room thermostat, programmer and TRVs" already_installed = "heating_control" in self.property.already_installed if already_installed: diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 103fa7b1..dc2bf1b8 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -137,8 +137,6 @@ class HeatingRecommender: if hhr_valid: # Recommend high heat retention storage heaters - # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just - # needs the controls self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable( @@ -474,12 +472,8 @@ class HeatingRecommender: } controls_description = controls_recommendations[0]['description'] - # Make the first letter of the description lowercase - controls_description = ( - controls_description[0].lower() + controls_description[1:] - ) - recommendation_description = f"{description} and {controls_description}" + recommendation_description = f"{description} {controls_description}" already_installed = "heating_controls" in self.property.already_installed if already_installed: @@ -555,6 +549,14 @@ class HeatingRecommender: We will recommend upgrading to a high heat retention storage system, if the current system is not already high heat retention storage + If the property currently has electric storage heaters, with automatic charge control, we allow for a high + heat retention stoarage heaters recommendation. This is because the automatic charge control is not the same + as the high heat retention storage heaters. HHR storage heaters aren't guaranteed to be more efficient but + we can at least present the option to the end user and they can decide if they want to go ahead with the + recommendation or not. There's a useful guide by quidos, describing the differences between some of the + different storage heater options: + https://www.quidos.co.uk/wp-content/uploads/2017/04/Technical-Bulletin-010417-Storage-Heatersv2.pdf + :param phase: The phase of the recommendation :param system_change: Indicates if we are recommending a different type of heating system, compared to the current system @@ -600,7 +602,15 @@ class HeatingRecommender: costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms ) - description = "Install high heat retention electric storage heaters" + description = "Install high heat retention electric storage heaters." + + # We check the existing heating system and controls + if ( + self.property.main_heating["has_electric_storage_heaters"] and + self.property.main_heating_controls["charging_system"] in ["automatic charge control"] + ): + description += (" The current electric heaters may be retrofit with high heat retention storage controls" + " however this is dependent on the existing system and may not be possible.") heating_description_simulation = { "mainheat-description": new_heating_description, diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index c64aab6f..0281a26d 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -144,11 +144,20 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 6.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100090311351.0, 'uprn-source': 'Address Matched', 'property-type_y': None, 'built-form_y': None, }, - "heating_recommendation_descriptions": [], + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' + 'high heat retention storage controls however this is dependent on the existing system and may not be ' + 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + + ], "heating_controls_recommendation_descriptions": [], - "notes": "This test has electric storage heaters with automatic charge control - this case should be researched" - "and checked that a high heat retention storage recommendation is actually sensible. If it's not, " - "we should adjust accordingly or perhaps have just a control recommendation" + "notes": "This test has electric storage heaters with automatic charge control - we recommend hhr storage" + "heaters in this case, but because there are already electic storage heaters in place, we " + "note, in the description of the recommendation, that this upgrade may be possible by retrofitting" + "the existing storage heaters, but that dependes on the model of the existing heaters" }, { "epc": { @@ -188,9 +197,18 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [], - "heating_controls_recommendation_descriptions": [], - "notes": "" + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler' + ], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], + "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to " + "the current water heating efficiency) the only recommendation we expect is for " + "a boiler upgrade. The heating controls are programmer and thermostat, so we can also recommend" + "better heating controls" }, { "epc": { diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 968583e4..4351623b 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -54,13 +54,6 @@ class TestHeatingRecommendations: :return: """ - if test_case["epc"]["uprn"] == 100090311351: - raise Exception( - "This test has electric storage heaters with automatic charge control - this case should be researched" - "and checked that a high heat retention storage recommendation is actually sensible. If it's not, " - "we should adjust accordingly or perhaps have just a control recommendation" - ) - if test_case["epc"]["uprn"] == 100021560521: raise Exception("Finish this test - could do so while on the train") From 342f926415f5c4239386b1081a3b43e37fb5a99a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 13:57:36 +0100 Subject: [PATCH 06/59] Added another unit test for heating recommendations --- backend/Property.py | 10 +++++++++- .../tests/test_data/heating_recommendations_data.py | 13 ++++++++++--- .../tests/test_heating_recommendations.py | 3 --- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 704e4f0a..b15c9f25 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1224,7 +1224,15 @@ class Property: if "air_source_heat_pump" not in measures: return False - suitable_property_type = self.data["property-type"] in ["House", "Bungalow"] + suitable_house = self.data["property-type"] == "House" and self.data["built-form"] in [ + "Detached", "Semi-Detached", + ] + + suitable_bungalow = self.data["property-type"] == "Bungalow" and self.data["built-form"] in [ + "Detached", "Semi-Detached" + ] + + suitable_property_type = suitable_house or suitable_bungalow has_air_source_heat_pump = self.main_heating["has_air_source_heat_pump"] return suitable_property_type and not has_air_source_heat_pump diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 0281a26d..78f6c7bf 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -405,11 +405,18 @@ testing_examples = [ 'construction-age-band': 'England and Wales: 1930-1949', 'lodgement-datetime': '2013-07-23 14:20:29', 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 9.0, 'uprn': 100070358594, - 'uprn-source': 'Address Matched' + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [], + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Upgrade to a new condensing boiler upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)' + ], "heating_controls_recommendation_descriptions": [], - "notes": "" + "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." + "We can recommend a boiler upgrade and high heat retention storage heaters" } ] diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 4351623b..35373729 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -54,9 +54,6 @@ class TestHeatingRecommendations: :return: """ - if test_case["epc"]["uprn"] == 100021560521: - raise Exception("Finish this test - could do so while on the train") - epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} epc_record = EPCRecord( From 7e11584407904e9bc6936e73762211315ac6da3d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:08:28 +0100 Subject: [PATCH 07/59] Added coverage for oil boiler --- recommendations/HeatingRecommender.py | 14 ++++- .../test_data/heating_recommendations_data.py | 56 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index dc2bf1b8..3a5e0c2c 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -27,6 +27,11 @@ class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" + # These are descriptions for boilers that are not gas boilers + NON_GAS_BOILERS = [ + "Boiler and radiators, oil", + ] + def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -84,13 +89,20 @@ class HeatingRecommender: self.property.data["mains-gas-flag"] ) + # The next condition is if the home has a non-gas boiler, such as an oil boiler + non_gas_boiler = ( + self.property.main_heating["clean_description"] in self.NON_GAS_BOILERS and + self.property.data["mains-gas-flag"] + ) + is_valid = ( ( has_boiler or no_heating_has_mains or electic_heating_has_mains or has_room_heaters or - portable_heaters_has_mains + portable_heaters_has_mains or + non_gas_boiler ) and (not ashp_only_heating_recommendation) and ("boiler_upgrade" in measures) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 78f6c7bf..343c0600 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -417,6 +417,53 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." "We can recommend a boiler upgrade and high heat retention storage heaters" + }, + { + "epc": { + 'lmk-key': '1162853989402014062718391220442948', 'address1': '145, Darley Green Road', 'address2': 'Knowle', + 'address3': None, 'postcode': 'B93 8PU', 'building-reference-number': 4475684278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'D', 'current-energy-efficiency': 23, + 'potential-energy-efficiency': 58, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2014-06-24', 'local-authority': 'E08000029', 'constituency': 'E14000812', + 'county': None, + 'lodgement-date': '2014-06-27', 'transaction-type': 'none of the above', 'environment-impact-current': 17, + 'environment-impact-potential': 45, 'energy-consumption-current': 382, 'energy-consumption-potential': 194, + 'co2-emissions-current': 27.0, 'co2-emiss-curr-per-floor-area': 94, 'co2-emissions-potential': 14.0, + 'lighting-cost-current': 175, 'lighting-cost-potential': 106, 'heating-cost-current': 5477, + 'heating-cost-potential': 3001, 'hot-water-cost-current': 267, 'hot-water-cost-potential': 120, + 'total-floor-area': 293.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2106.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 2, + 'number-habitable-rooms': 12, 'number-heated-rooms': 12, 'low-energy-lighting': 31, + 'number-open-fireplaces': 2, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Average', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', + 'windows-env-eff': 'Very Poor', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Boiler and radiators, oil', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 31% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'oil (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.5, 'photo-supply': 0.0, 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '145, Darley Green Road, Knowle', + 'local-authority-label': 'Solihull', 'constituency-label': 'Meriden', 'posttown': 'SOLIHULL', + 'construction-age-band': 'England and Wales: before 1900', + 'lodgement-datetime': '2014-06-27 18:39:12', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': 42.0, 'low-energy-fixed-light-count': 13.0, 'uprn': 100070985545, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" + "an air source heat pump" } ] @@ -460,11 +507,18 @@ print(eg["built-form"]) print(eg["mainheatcont-description"]) ### We also use the Midlands EPC F/G portfolio to get examples to create tests + +completed_descriptions = [ + "Portable electric heaters assumed for most rooms" +] + portfolio = pd.read_excel( "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] + eg = portfolio[ - (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms") + (portfolio["mainheat-description"] == "Boiler and radiators, oil") ].sample(1) eg = eg.squeeze().to_dict() From f86a7cb97d97bdc903509c18a3c5d784df00b3b9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:43:21 +0100 Subject: [PATCH 08/59] adding heating unit test coverage --- recommendations/HeatingRecommender.py | 19 +- .../test_data/heating_recommendations_data.py | 171 +++++++++++++++++- 2 files changed, 171 insertions(+), 19 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 3a5e0c2c..5d68ebcf 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -14,13 +14,6 @@ class HeatingRecommender: "No system present, electric heaters assumed" ] - ELECTRIC_HEATING_DESCRIPTIONS = [ - "Room heaters, electric", - "Electric storage heaters", - "Electric storage heaters, radiators", - "Portable electric heaters assumed for most rooms", - ] - ROOM_HEATERS_DESCRIPTIONS = [ "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", ] @@ -39,9 +32,7 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = self.property.main_heating["clean_description"] in ( - self.ELECTRIC_HEATING_DESCRIPTIONS + self.ASSUMED_ELECTRIC_HEATING - ) + self.has_electric_heating_description = self.property.main_heating["has_electric"] def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ @@ -49,14 +40,8 @@ class HeatingRecommender: :return: """ - # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we - # can consider hhr storage heaters - electric_heating_assumed = self.property.main_heating["clean_description"] in self.ASSUMED_ELECTRIC_HEATING - - has_electric = self.has_electric_heating_description or electric_heating_assumed - return ( - has_electric and (not ashp_only_heating_recommendation) and + self.has_electric_heating_description and (not ashp_only_heating_recommendation) and ("high_heat_retention_storage_heater" in measures) ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 343c0600..eec7703e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -464,6 +464,168 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" "an air source heat pump" + }, + { + "epc": { + 'lmk-key': '351990902052009082517013406210567', 'address1': '56, Collingham Road', 'address2': None, + 'address3': None, 'postcode': 'LE3 2BA', 'building-reference-number': 5783266668, + 'current-energy-rating': 'F', 'potential-energy-rating': 'F', 'current-energy-efficiency': 28, + 'potential-energy-efficiency': 33, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2009-08-25', 'local-authority': 'E06000016', 'constituency': 'E14000784', + 'county': None, + 'lodgement-date': '2009-08-25', 'transaction-type': 'marketed sale', 'environment-impact-current': 31, + 'environment-impact-potential': 33, 'energy-consumption-current': 579, 'energy-consumption-potential': 549, + 'co2-emissions-current': 7.4, 'co2-emiss-curr-per-floor-area': 95, 'co2-emissions-potential': 7.1, + 'lighting-cost-current': 78, 'lighting-cost-potential': 39, 'heating-cost-current': 985, + 'heating-cost-potential': 1015, 'hot-water-cost-current': 381, 'hot-water-cost-potential': 281, + 'total-floor-area': 87.8, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 35.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 2, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Partial double glazing', 'windows-energy-eff': 'Poor', + 'windows-env-eff': 'Poor', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, no insulation (assumed)', + 'roof-energy-eff': 'Very Poor', 'roof-env-eff': 'Very Poor', + 'mainheat-description': 'Room heaters, mains gas', 'mainheat-energy-eff': 'Average', + 'mainheat-env-eff': 'Average', 'mainheatcont-description': 'No thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.48, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '56, Collingham Road', 'local-authority-label': 'Leicester', + 'constituency-label': 'Leicester West', 'posttown': 'LEICESTER', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2009-08-25 17:01:34', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 2465031849, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" + "well as an air source heat pump" + }, + { + "epc": { + 'lmk-key': 'f9997a382dca2a1b5dc916a21cf1a28327fc6ffe32fa4c5eeb7a859fe73cabf4', + 'address1': '39 Parkes Street', 'address2': None, 'address3': None, 'postcode': 'WV13 2LR', + 'building-reference-number': 10005271458, 'current-energy-rating': 'G', 'potential-energy-rating': 'B', + 'current-energy-efficiency': 17, 'potential-energy-efficiency': 89, 'property-type': 'House', + 'built-form': 'Mid-Terrace', 'inspection-date': '2023-11-10', 'local-authority': 'E08000030', + 'constituency': 'E14001011', 'county': None, 'lodgement-date': '2023-11-10', 'transaction-type': 'rental', + 'environment-impact-current': 29, 'environment-impact-potential': 88, 'energy-consumption-current': 582, + 'energy-consumption-potential': 71, 'co2-emissions-current': 7.4, 'co2-emiss-curr-per-floor-area': 98, + 'co2-emissions-potential': 1.0, 'lighting-cost-current': 193, 'lighting-cost-potential': 121, + 'heating-cost-current': 3789, 'heating-cost-potential': 774, 'hot-water-cost-current': 1241, + 'hot-water-cost-potential': 165, 'total-floor-area': 75.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 0, + 'low-energy-lighting': 40, 'number-open-fireplaces': 0, + 'hotwater-description': 'No system present: electric immersion assumed', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'No system present: electric heaters assumed', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Poor', 'mainheatcont-description': 'None', 'mainheatc-energy-eff': 'Very Poor', + 'mainheatc-env-eff': 'Very Poor', 'lighting-description': 'Low energy lighting in 40% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', + 'main-fuel': 'To be used only when there is no heating/hot-water system or data is from a community ' + 'network', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.5, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '39 Parkes Street', 'local-authority-label': 'Walsall', 'constituency-label': 'Walsall North', + 'posttown': 'WILLENHALL', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2023-11-10 18:06:18', 'tenure': 'Rented (social)', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': None, 'uprn': 100071113763, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has assumed electric heaters. Boiler upgrade, HHR and ASHP are all recommended" + }, + { + "epc": { + 'lmk-key': 'dca62e7f9e21ac21d9c8af1029102cbe47c0509b8e6fc302cd7df079f8fc3a53', + 'address1': '58 Telford Road', 'address2': None, 'address3': None, 'postcode': 'WS2 7LD', + 'building-reference-number': 10003473159, 'current-energy-rating': 'F', 'potential-energy-rating': 'C', + 'current-energy-efficiency': 37, 'potential-energy-efficiency': 77, 'property-type': 'House', + 'built-form': 'Mid-Terrace', 'inspection-date': '2022-10-26', 'local-authority': 'E08000030', + 'constituency': 'E14001011', 'county': None, 'lodgement-date': '2022-10-26', + 'transaction-type': 'marketed sale', 'environment-impact-current': 47, 'environment-impact-potential': 80, + 'energy-consumption-current': 380, 'energy-consumption-potential': 130, 'co2-emissions-current': 4.5, + 'co2-emiss-curr-per-floor-area': 65, 'co2-emissions-potential': 1.6, 'lighting-cost-current': 101, + 'lighting-cost-potential': 63, 'heating-cost-current': 1267, 'heating-cost-potential': 733, + 'hot-water-cost-current': 386, 'hot-water-cost-potential': 67, 'total-floor-area': 69.0, + 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': None, 'flat-top-storey': None, + 'flat-storey-count': None, 'main-heating-controls': None, 'multi-glaze-proportion': 90.0, + 'glazed-type': 'double glazing, unknown install date', 'glazed-area': 'Normal', 'extension-count': 0, + 'number-habitable-rooms': 4, 'number-heated-rooms': 1, 'low-energy-lighting': 40, + 'number-open-fireplaces': 0, 'hotwater-description': 'Electric immersion, standard tariff', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Mostly double glazing', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Cavity wall, filled cavity', + 'walls-energy-eff': 'Average', 'walls-env-eff': 'Average', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, 100 mm loft insulation', + 'roof-energy-eff': 'Average', 'roof-env-eff': 'Average', + 'mainheat-description': 'Portable electric heaters assumed for most rooms, Room heaters, electric', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Poor', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 40% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'mains gas (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.46, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '58 Telford Road', 'local-authority-label': 'Walsall', 'constituency-label': 'Walsall North', + 'posttown': 'WALSALL', 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2022-10-26 13:46:05', 'tenure': 'Owner-occupied', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': None, 'uprn': 100071089116, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + ], + "heating_controls_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" + "upgrade and ASHP" } ] @@ -509,7 +671,11 @@ print(eg["mainheatcont-description"]) ### We also use the Midlands EPC F/G portfolio to get examples to create tests completed_descriptions = [ - "Portable electric heaters assumed for most rooms" + "Portable electric heaters assumed for most rooms", + "Boiler and radiators, oil", + "Boiler and radiators, mains gas", + "Room heaters, mains gas", + "No system present: electric heaters assumed" ] portfolio = pd.read_excel( @@ -517,8 +683,9 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, oil") + (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms, Room heaters, electric") ].sample(1) eg = eg.squeeze().to_dict() From e31adcb55c48f66195a9aa31a532a48ff4183a61 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:50:52 +0100 Subject: [PATCH 09/59] Added addition mainheat unit tests --- recommendations/HeatingRecommender.py | 3 +- .../test_data/heating_recommendations_data.py | 56 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 5d68ebcf..901add15 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -604,7 +604,8 @@ class HeatingRecommender: # We check the existing heating system and controls if ( self.property.main_heating["has_electric_storage_heaters"] and - self.property.main_heating_controls["charging_system"] in ["automatic charge control"] + self.property.main_heating_controls["charging_system"] in + ["automatic charge control", "manual charge control"] ): description += (" The current electric heaters may be retrofit with high heat retention storage controls" " however this is dependent on the existing system and may not be possible.") diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index eec7703e..5b2b4aa0 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -626,6 +626,57 @@ testing_examples = [ ], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" + }, + { + "epc": { + 'lmk-key': '594183609042011021816283787499688', 'address1': '96, Richmond Road', 'address2': None, + 'address3': None, 'postcode': 'DE23 8PX', 'building-reference-number': 54104868, + 'current-energy-rating': 'F', 'potential-energy-rating': 'F', 'current-energy-efficiency': 30, + 'potential-energy-efficiency': 31, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2011-02-18', 'local-authority': 'E06000015', 'constituency': 'E14000663', + 'county': None, + 'lodgement-date': '2011-02-18', 'transaction-type': 'rental (social)', 'environment-impact-current': 25, + 'environment-impact-potential': 26, 'energy-consumption-current': 709, 'energy-consumption-potential': 693, + 'co2-emissions-current': 8.7, 'co2-emiss-curr-per-floor-area': 107, 'co2-emissions-potential': 8.5, + 'lighting-cost-current': 56, 'lighting-cost-potential': 56, 'heating-cost-current': 1118, + 'heating-cost-potential': 1089, 'hot-water-cost-current': 164, 'hot-water-cost-potential': 164, + 'total-floor-area': 80.98, 'energy-tariff': 'dual', 'mains-gas-flag': 'Y', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2401.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, + 'low-energy-lighting': 88, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, off-peak', 'hot-water-energy-eff': 'Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, electric', 'roof-description': 'Pitched, 150 mm loft insulation', + 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', 'mainheat-description': 'Electric storage heaters', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'Manual charge control', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 88% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'electricity - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.72, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '96, Richmond Road', 'local-authority-label': 'Derby', + 'constituency-label': 'Derby South', 'posttown': 'DERBY', + 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2011-02-18 16:28:37', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030352255, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' + 'high heat retention storage controls however this is dependent on the existing system and may not be ' + 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" + "the ashp is not suitable" } ] @@ -675,7 +726,8 @@ completed_descriptions = [ "Boiler and radiators, oil", "Boiler and radiators, mains gas", "Room heaters, mains gas", - "No system present: electric heaters assumed" + "No system present: electric heaters assumed", + "Room heaters, electric", ] portfolio = pd.read_excel( @@ -686,6 +738,6 @@ portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descript portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms, Room heaters, electric") + (portfolio["mainheat-description"] == "Electric storage heaters") ].sample(1) eg = eg.squeeze().to_dict() From 9156349ee592cfa3adde1c2b03d94d720338eab4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 17:34:28 +0100 Subject: [PATCH 10/59] Increasing heating system reommendations --- recommendations/HeatingRecommender.py | 9 +- .../test_data/heating_recommendations_data.py | 104 +++++++++++++++++- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 901add15..5f632567 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -23,6 +23,8 @@ class HeatingRecommender: # These are descriptions for boilers that are not gas boilers NON_GAS_BOILERS = [ "Boiler and radiators, oil", + "Boiler and radiators, lpg", + "Boiler and radiators, electric" ] def __init__(self, property_instance: Property): @@ -40,8 +42,13 @@ class HeatingRecommender: :return: """ + # We can also recommend hhr if the property doesn't have a mains has connection + no_mains = not self.property.data["mains-gas-flag"] + + hhr_suitable = no_mains or self.has_electric_heating_description + return ( - self.has_electric_heating_description and (not ashp_only_heating_recommendation) and + hhr_suitable and (not ashp_only_heating_recommendation) and ("high_heat_retention_storage_heater" in measures) ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 5b2b4aa0..bebbfec9 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -677,6 +677,106 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" "the ashp is not suitable" + }, + { + "epc": { + 'lmk-key': '206883665252008121709314507989954', 'address1': '2 Upper Gardens Hazelhurst', + 'address2': 'Bishopswood', 'address3': None, 'postcode': 'HR9 5QX', 'building-reference-number': 9443575568, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 32, + 'potential-energy-efficiency': 42, 'property-type': 'Bungalow', 'built-form': 'Semi-Detached', + 'inspection-date': '2008-12-11', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2008-12-17', 'transaction-type': 'rental (private)', 'environment-impact-current': 54, + 'environment-impact-potential': 64, 'energy-consumption-current': 290, 'energy-consumption-potential': 231, + 'co2-emissions-current': 3.7, 'co2-emiss-curr-per-floor-area': 58, 'co2-emissions-potential': 2.9, + 'lighting-cost-current': 39, 'lighting-cost-potential': 39, 'heating-cost-current': 654, + 'heating-cost-potential': 515, 'hot-water-cost-current': 204, 'hot-water-cost-potential': 169, + 'total-floor-area': 63.94, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2106.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 3, + 'low-energy-lighting': 75, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Average', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, insulated (assumed)', 'walls-energy-eff': 'Good', + 'walls-env-eff': 'Good', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 100 mm loft insulation', 'roof-energy-eff': 'Average', + 'roof-env-eff': 'Average', 'mainheat-description': 'Boiler and radiators, LPG', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Average', + 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 75% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'LPG - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.4, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '2 Upper Gardens Hazelhurst, Bishopswood', + 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'ROSS-ON-WYE', + 'construction-age-band': 'England and Wales: 1983-1990', + 'lodgement-datetime': '2008-12-17 09:31:45', 'tenure': 'rental (private)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10009573249, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has an LFG boiler but it doesn't have a mains gas connection so we can only recommend" + "an air source heat pump and hhr storage" + }, + { + "epc": { + 'lmk-key': '749e6ae968c0d5c6491ee1ee82a591733568c981e08ffde7e92b6e4172f3fb0f', + 'address1': '10 Small Holdings', 'address2': 'Stoneleigh Road', 'address3': 'Baginton', + 'postcode': 'CV8 3BA', 'building-reference-number': 10005704940, 'current-energy-rating': 'G', + 'potential-energy-rating': 'E', 'current-energy-efficiency': 12, 'potential-energy-efficiency': 46, + 'property-type': 'House', 'built-form': 'Semi-Detached', 'inspection-date': '2024-03-04', + 'local-authority': 'E07000222', 'constituency': 'E14000767', 'county': 'Warwickshire', + 'lodgement-date': '2024-03-08', 'transaction-type': 'not sale or rental', 'environment-impact-current': 14, + 'environment-impact-potential': 39, 'energy-consumption-current': 728, 'energy-consumption-potential': 379, + 'co2-emissions-current': 11.0, 'co2-emiss-curr-per-floor-area': 137, 'co2-emissions-potential': 5.9, + 'lighting-cost-current': 169, 'lighting-cost-potential': 169, 'heating-cost-current': 4435, + 'heating-cost-potential': 2718, 'hot-water-cost-current': 405, 'hot-water-cost-potential': 227, + 'total-floor-area': 81.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': None, + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'not defined', 'glazed-area': 'Much More Than Typical', + 'extension-count': 0, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, 'low-energy-lighting': 63, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, smokeless fuel', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Boiler and radiators, electric', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Poor', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 63% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.31, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '10 Small Holdings, Stoneleigh Road, Baginton', 'local-authority-label': 'Warwick', + 'constituency-label': 'Kenilworth and Southam', 'posttown': 'COVENTRY', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2024-03-08 10:43:35', 'tenure': 'Rented (social)', + 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 10013181470, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " + "recommend a boiler upgrade. We recommend HHR and ASHP" } ] @@ -728,6 +828,8 @@ completed_descriptions = [ "Room heaters, mains gas", "No system present: electric heaters assumed", "Room heaters, electric", + "Electric storage heaters", + "Boiler and radiators, LPG", ] portfolio = pd.read_excel( @@ -738,6 +840,6 @@ portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descript portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Electric storage heaters") + (portfolio["mainheat-description"] == "Boiler and radiators, electric") ].sample(1) eg = eg.squeeze().to_dict() From c086ecae5285836858bc941482d3c9a2e7319798 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:32:02 +0100 Subject: [PATCH 11/59] Additional heating recommendations covered and tests added --- etl/customers/aiha/epc_data_pull.py | 20 +++- recommendations/HeatingRecommender.py | 17 +-- .../test_data/heating_recommendations_data.py | 106 +++++++++++++++++- 3 files changed, 129 insertions(+), 14 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index f7f4631c..8259578d 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -676,6 +676,24 @@ def app(): "Manchester", archetyping_data["Location"] ) + # We fix the location for B 80 Bethune Road + archetyping_data["Location"] = np.where( + ( + archetyping_data["row_id"].isin( + data[ + data["Street address"] == "80 Bethune Road" + ]["row_id"].values.tolist() + ) + ) & ( + archetyping_data["row_id"].isin( + data[ + data["Address letter or number"] == "B" + ]["row_id"].values.tolist() + ) + ), + "London", + archetyping_data["Location"] + ) # Hackney 73 - London # Southend-on-Sea 6 - Southend @@ -691,7 +709,7 @@ def app(): 'Current heating system type', 'Wall type', 'Roof type', - "Location", + # "Location", # 'current-energy-rating', 'property-type-reduced', 'built-form-reduced', 'is_cavity_wall', # 'is_solid_brick_wall', 'is_system_built_wall', 'is_timber_frame_wall', 'is_as_built', # 'is_solid', 'is_roof_room', diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 5f632567..64a2e285 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -20,13 +20,6 @@ class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" - # These are descriptions for boilers that are not gas boilers - NON_GAS_BOILERS = [ - "Boiler and radiators, oil", - "Boiler and radiators, lpg", - "Boiler and radiators, electric" - ] - def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -59,7 +52,8 @@ class HeatingRecommender: """ # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls - has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"] + # If it's NOT a gas boiler, we'll potentially recommend a boiler + has_gas_boiler = self.property.main_heating["has_boiler"] and self.property.main_heating["has_mains_gas"] # 2) If the property doesn't have a heating system, but it has access to the mains gas no_heating_has_mains = self.property.main_heating["clean_description"] in [ @@ -81,15 +75,16 @@ class HeatingRecommender: self.property.data["mains-gas-flag"] ) - # The next condition is if the home has a non-gas boiler, such as an oil boiler + # The next condition is if the home has a non-gas boiler, such as an oil boiler, with a mains gas connection non_gas_boiler = ( - self.property.main_heating["clean_description"] in self.NON_GAS_BOILERS and + self.property.main_heating["has_boiler"] and + not self.property.main_heating["has_mains_gas"] and self.property.data["mains-gas-flag"] ) is_valid = ( ( - has_boiler or + has_gas_boiler or no_heating_has_mains or electic_heating_has_mains or has_room_heaters or diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index bebbfec9..2fc47e13 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -777,6 +777,105 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " "recommend a boiler upgrade. We recommend HHR and ASHP" + }, + { + "epc": { + 'lmk-key': '683441359142011092814474999092088', 'address1': '20, Haybridge Avenue', 'address2': 'Hadley', + 'address3': None, 'postcode': 'TF1 5JR', 'building-reference-number': 3100250968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 34, + 'potential-energy-efficiency': 41, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2011-09-28', 'local-authority': 'E06000020', 'constituency': 'E14000992', + 'county': None, + 'lodgement-date': '2011-09-28', 'transaction-type': 'rental (social)', 'environment-impact-current': 29, + 'environment-impact-potential': 34, 'energy-consumption-current': 495, 'energy-consumption-potential': 435, + 'co2-emissions-current': 8.3, 'co2-emiss-curr-per-floor-area': 97, 'co2-emissions-potential': 7.3, + 'lighting-cost-current': 61, 'lighting-cost-potential': 45, 'heating-cost-current': 1273, + 'heating-cost-potential': 1101, 'hot-water-cost-current': 214, 'hot-water-cost-potential': 214, + 'total-floor-area': 85.1, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2101.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 1, + 'number-habitable-rooms': 5, 'number-heated-rooms': 5, 'low-energy-lighting': 64, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system, no cylinder thermostat', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', 'windows-env-eff': 'Very Poor', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 250mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in 64% of fixed outlets', 'lighting-energy-eff': 'Good', + 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, + 'photo-supply': 0.0, + 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', + 'address': '20, Haybridge Avenue, Hadley', 'local-authority-label': 'Telford and Wrekin', + 'constituency-label': 'The Wrekin', 'posttown': 'TELFORD', + 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2011-09-28 14:47:49', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': 7.0, 'uprn': 452047507, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a dual fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': 'ba1de1b99f30546d7c6654af44c74fa4511611f9283502b77efb825d8566023c', + 'address1': '19 DORSET STREET', 'address2': 'DERBY', 'address3': None, 'postcode': 'DE21 6BE', + 'building-reference-number': 10000116666, 'current-energy-rating': 'F', 'potential-energy-rating': 'C', + 'current-energy-efficiency': 29, 'potential-energy-efficiency': 78, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-01-09', 'local-authority': 'E06000015', + 'constituency': 'E14000662', 'county': None, 'lodgement-date': '2021-01-11', + 'transaction-type': 'ECO assessment', 'environment-impact-current': 1, 'environment-impact-potential': 40, + 'energy-consumption-current': 532, 'energy-consumption-potential': 153, 'co2-emissions-current': 14.0, + 'co2-emiss-curr-per-floor-area': 198, 'co2-emissions-potential': 5.1, 'lighting-cost-current': 105, + 'lighting-cost-potential': 60, 'heating-cost-current': 1361, 'heating-cost-potential': 545, + 'hot-water-cost-current': 242, 'hot-water-cost-potential': 132, 'total-floor-area': 72.0, + 'energy-tariff': 'off-peak 7 hour', 'mains-gas-flag': 'N', 'floor-level': None, 'flat-top-storey': None, + 'flat-storey-count': None, 'main-heating-controls': None, 'multi-glaze-proportion': 100.0, + 'glazed-type': 'double glazing, unknown install date', 'glazed-area': 'Normal', 'extension-count': 0, + 'number-habitable-rooms': 4, 'number-heated-rooms': 4, 'low-energy-lighting': 25, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Average', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, coal', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in 25% of fixed outlets', 'lighting-energy-eff': 'Average', + 'lighting-env-eff': 'Average', 'main-fuel': 'house coal (not community)', 'wind-turbine-count': 0, + 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.37, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '19 DORSET STREET, DERBY', + 'local-authority-label': 'Derby', 'constituency-label': 'Derby North', 'posttown': 'DERBY', + 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2021-01-11 00:00:00', 'tenure': 'Owner-occupied', + 'fixed-lighting-outlets-count': 16.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100030309413, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" } ] @@ -830,6 +929,8 @@ completed_descriptions = [ "Room heaters, electric", "Electric storage heaters", "Boiler and radiators, LPG", + "Boiler and radiators, electric", + "Boiler and radiators, dual fuel (mineral and wood)" ] portfolio = pd.read_excel( @@ -837,9 +938,10 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -portfolio["mainheat-description"].value_counts() +print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, electric") + (portfolio["mainheat-description"] == "Boiler and radiators, coal") ].sample(1) eg = eg.squeeze().to_dict() +print(eg) From 64d19defbb7c3e6c9aff990ba533bffa26cfd787 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:36:44 +0100 Subject: [PATCH 12/59] more heating tests --- .../test_data/heating_recommendations_data.py | 112 +++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 2fc47e13..982eb280 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -876,6 +876,108 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1139832199022019020816400153188351', 'address1': '1 Green Gates', 'address2': 'Bridstow', + 'address3': None, 'postcode': 'HR9 6QJ', 'building-reference-number': 5576913278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'A', 'current-energy-efficiency': 37, + 'potential-energy-efficiency': 93, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2019-02-08', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2019-02-08', 'transaction-type': 'ECO assessment', 'environment-impact-current': 11, + 'environment-impact-potential': 115, 'energy-consumption-current': 377, 'energy-consumption-potential': 28, + 'co2-emissions-current': 14.0, 'co2-emiss-curr-per-floor-area': 129, 'co2-emissions-potential': -1.9, + 'lighting-cost-current': 75, 'lighting-cost-potential': 75, 'heating-cost-current': 1512, + 'heating-cost-potential': 700, 'hot-water-cost-current': 258, 'hot-water-cost-potential': 113, + 'total-floor-area': 111.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2101.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 100, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Average', 'hot-water-env-eff': 'Very Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 270 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, smokeless fuel', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good', + 'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow', + 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'ROSS-ON-WYE', + 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2019-02-08 16:40:01', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10007366417, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a smokeless fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1253529329242015021115045635159198', 'address1': '143', 'address2': 'Shortheath', + 'address3': None, 'postcode': 'DE12 6BL', 'building-reference-number': 212621378, + 'current-energy-rating': 'F', 'potential-energy-rating': 'D', 'current-energy-efficiency': 22, + 'potential-energy-efficiency': 59, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2015-02-11', 'local-authority': 'E07000039', 'constituency': 'E14000935', + 'county': 'Derbyshire', 'lodgement-date': '2015-02-11', 'transaction-type': 'RHI application', + 'environment-impact-current': 71, 'environment-impact-potential': 91, 'energy-consumption-current': 500, + 'energy-consumption-potential': 233, 'co2-emissions-current': 3.0, 'co2-emiss-curr-per-floor-area': 31, + 'co2-emissions-potential': 0.8, 'lighting-cost-current': 104, 'lighting-cost-potential': 59, + 'heating-cost-current': 1746, 'heating-cost-potential': 1010, 'hot-water-cost-current': 253, + 'hot-water-cost-potential': 151, 'total-floor-area': 96.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', + 'floor-level': 'NODATA!', 'flat-top-storey': None, 'flat-storey-count': None, + 'main-heating-controls': 2111.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 23, 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Very Good', + 'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 250 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, wood pellets', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Good', 'mainheatcont-description': 'TRVs and bypass', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor', + 'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath', + 'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire', + 'posttown': 'SWADLINCOTE', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2015-02-11 15:04:56', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030256931, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" } ] @@ -930,7 +1032,9 @@ completed_descriptions = [ "Electric storage heaters", "Boiler and radiators, LPG", "Boiler and radiators, electric", - "Boiler and radiators, dual fuel (mineral and wood)" + "Boiler and radiators, dual fuel (mineral and wood)", + "Boiler and radiators, coal", + "Boiler and radiators, smokeless fuel" ] portfolio = pd.read_excel( @@ -938,10 +1042,14 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +portfolio['sheating-energy-eff'] = None +portfolio['sheating-env-eff'] = None +portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) + print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, coal") + (portfolio["mainheat-description"] == "Boiler and radiators, wood pellets") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 942f1958db942513ea44e50e07191148fec6f38b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:42:30 +0100 Subject: [PATCH 13/59] another heating recommendation test --- recommendations/HeatingRecommender.py | 20 +++----- .../test_data/heating_recommendations_data.py | 51 ++++++++++++++++++- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 64a2e285..8fbcec86 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -9,15 +9,6 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: - ASSUMED_ELECTRIC_HEATING = [ - "Portable electric heaters assumed for most rooms", - "No system present, electric heaters assumed" - ] - - ROOM_HEATERS_DESCRIPTIONS = [ - "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", - ] - high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" def __init__(self, property_instance: Property): @@ -62,7 +53,10 @@ class HeatingRecommender: # The property is using portable heaters and has access to gas mains has_room_heaters = ( - self.property.main_heating["clean_description"] in self.ROOM_HEATERS_DESCRIPTIONS and + ( + self.property.main_heating["has_room_heaters"] or + self.property.main_heating["has_portable_electric_heaters"] + ) and self.property.data["mains-gas-flag"] ) @@ -95,7 +89,7 @@ class HeatingRecommender: ("boiler_upgrade" in measures) ) - return is_valid, has_boiler + return is_valid, has_gas_boiler def recommend(self, has_cavity_or_loft_recommendations, phase=0, measures=None): """ @@ -138,14 +132,14 @@ class HeatingRecommender: # Recommend high heat retention storage heaters self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) - gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable( + gas_boiler_suitable, has_gas_boiler = self.is_boiler_upgrade_suitable( measures=measures, ashp_only_heating_recommendation=ashp_only_heating_recommendation ) if gas_boiler_suitable: # This indicates that the home previously did not have a boiler in place and so would require # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler - system_change = not has_boiler + system_change = not has_gas_boiler exising_room_heaters = self.property.main_heating["clean_description"] in [ "Room heaters, electric", "Room heaters, mains gas" ] diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 982eb280..c6751784 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -978,6 +978,52 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1125990659062017030307353552528423', 'address1': '3 Manor Farm Cottage', 'address2': 'Halse', + 'address3': None, 'postcode': 'NN13 6DY', 'building-reference-number': 2529522278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'C', 'current-energy-efficiency': 29, + 'potential-energy-efficiency': 80, 'property-type': 'House', 'built-form': 'End-Terrace', + 'inspection-date': '2017-03-02', 'local-authority': 'E07000155', 'constituency': 'E14000942', + 'county': 'Northamptonshire', 'lodgement-date': '2017-03-03', 'transaction-type': 'rental (private)', + 'environment-impact-current': 26, 'environment-impact-potential': 74, 'energy-consumption-current': 511, + 'energy-consumption-potential': 130, 'co2-emissions-current': 8.0, 'co2-emiss-curr-per-floor-area': 108, + 'co2-emissions-potential': 2.2, 'lighting-cost-current': 84, 'lighting-cost-potential': 56, + 'heating-cost-current': 1333, 'heating-cost-potential': 498, 'hot-water-cost-current': 285, + 'hot-water-cost-potential': 144, 'total-floor-area': 74.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', + 'floor-level': 'NODATA!', 'flat-top-storey': None, 'flat-storey-count': None, + 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 1, + 'number-habitable-rooms': 4, 'number-heated-rooms': 1, 'low-energy-lighting': 50, + 'number-open-fireplaces': 0, 'hotwater-description': 'Electric immersion, off-peak', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', 'windows-env-eff': 'Very Poor', + 'walls-description': 'Sandstone or limestone, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 75 mm loft insulation', 'roof-energy-eff': 'Average', + 'roof-env-eff': 'Average', 'mainheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse', + 'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire', + 'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900', + 'lodgement-datetime': '2017-03-03 07:35:35', 'tenure': 'rental (private)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10000460605, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" } ] @@ -1034,7 +1080,8 @@ completed_descriptions = [ "Boiler and radiators, electric", "Boiler and radiators, dual fuel (mineral and wood)", "Boiler and radiators, coal", - "Boiler and radiators, smokeless fuel" + "Boiler and radiators, smokeless fuel", + "Boiler and radiators, wood pellets" ] portfolio = pd.read_excel( @@ -1049,7 +1096,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, wood pellets") + (portfolio["mainheat-description"] == "Room heaters, dual fuel (mineral and wood)") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 76154c51729aaf2b6da490f0f123226c462e47e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:48:25 +0100 Subject: [PATCH 14/59] another unit test covered --- recommendations/HeatingRecommender.py | 14 ++++-- .../test_data/heating_recommendations_data.py | 47 ++++++++++++++++++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 8fbcec86..bb074407 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -19,6 +19,7 @@ class HeatingRecommender: self.heating_control_recommendations = [] self.has_electric_heating_description = self.property.main_heating["has_electric"] + self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ @@ -31,8 +32,10 @@ class HeatingRecommender: hhr_suitable = no_mains or self.has_electric_heating_description + # If there's already an ASHP in place, we don't recommend HHR + return ( - hhr_suitable and (not ashp_only_heating_recommendation) and + hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and ("high_heat_retention_storage_heater" in measures) ) @@ -86,7 +89,8 @@ class HeatingRecommender: non_gas_boiler ) and (not ashp_only_heating_recommendation) and - ("boiler_upgrade" in measures) + ("boiler_upgrade" in measures) and + (not self.has_ashp) ) return is_valid, has_gas_boiler @@ -155,7 +159,11 @@ class HeatingRecommender: # In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions # and either allow or prevent the recommendation of an air source heat pump - if self.property.is_ashp_valid(measures=measures) and non_invasive_ashp_recommendation["suitable"]: + if ( + self.property.is_ashp_valid(measures=measures) and + non_invasive_ashp_recommendation["suitable"] and + not self.has_ashp + ): self.recommend_air_source_heat_pump( phase=phase, has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations, diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index c6751784..51d0636e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1024,6 +1024,48 @@ testing_examples = [ ], "heating_controls_recommendation_descriptions": [], "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" + }, + { + "epc": { + 'lmk-key': '1281510829102015021321472533359578', 'address1': '6, Nags Head Lane', 'address2': 'Hargrave', + 'address3': None, 'postcode': 'NN9 6BJ', 'building-reference-number': 134423378, + 'current-energy-rating': 'F', 'potential-energy-rating': 'B', 'current-energy-efficiency': 38, + 'potential-energy-efficiency': 84, 'property-type': 'House', 'built-form': 'End-Terrace', + 'inspection-date': '2015-02-13', 'local-authority': 'E07000152', 'constituency': 'E14000648', + 'county': 'Northamptonshire', 'lodgement-date': '2015-02-13', + 'transaction-type': 'assessment for green deal', 'environment-impact-current': 45, + 'environment-impact-potential': 85, 'energy-consumption-current': 400, 'energy-consumption-potential': 96, + 'co2-emissions-current': 5.0, 'co2-emiss-curr-per-floor-area': 68, 'co2-emissions-potential': 1.2, + 'lighting-cost-current': 87, 'lighting-cost-potential': 48, 'heating-cost-current': 1094, + 'heating-cost-potential': 423, 'hot-water-cost-current': 240, 'hot-water-cost-potential': 144, + 'total-floor-area': 74.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2204.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 18, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 300 mm loft insulation', 'roof-energy-eff': 'Very Good', + 'roof-env-eff': 'Very Good', 'mainheat-description': 'Air source heat pump, radiators, electric', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer and room thermostat', 'mainheatc-energy-eff': 'Average', + 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets', + 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave', + 'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby', + 'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2015-02-13 21:47:25', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100031045596, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [], + "notes": "This property already has an ashp. We don't recommend any heating upgrades" } ] @@ -1081,7 +1123,8 @@ completed_descriptions = [ "Boiler and radiators, dual fuel (mineral and wood)", "Boiler and radiators, coal", "Boiler and radiators, smokeless fuel", - "Boiler and radiators, wood pellets" + "Boiler and radiators, wood pellets", + "Room heaters, dual fuel (mineral and wood)", ] portfolio = pd.read_excel( @@ -1096,7 +1139,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, dual fuel (mineral and wood)") + (portfolio["mainheat-description"] == "Air source heat pump, radiators, electric") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 68fc8b8cbbba3f12e9081af92c2fd9ad0c4f0a94 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 09:15:31 +0100 Subject: [PATCH 15/59] Added dual heating combined recommendation --- recommendations/HeatingControlRecommender.py | 38 ++- recommendations/HeatingRecommender.py | 224 +++++++++++++++--- recommendations/recommendation_utils.py | 41 ++++ .../test_data/heating_recommendations_data.py | 97 +++++++- .../tests/test_heating_recommendations.py | 2 + 5 files changed, 360 insertions(+), 42 deletions(-) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 6f848441..62e292df 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -12,8 +12,11 @@ class HeatingControlRecommender: self.recommendation = [] - def recommend(self, heating_description): + def recommend(self, heating_description, description_prefix="", description_suffix=""): + # TODO: Many of these functions are quite similar. We can possibly create a single wrapper function that + # takes in the heating description and the description prefix/suffix, and then creates the appropriate + # output # Reset the recommendations self.recommendation = [] @@ -24,14 +27,14 @@ class HeatingControlRecommender: return if heating_description in ["Electric storage heaters", "Electric storage heaters, radiators"]: - self.recommend_high_heat_retention_controls() + self.recommend_high_heat_retention_controls(description_prefix=description_prefix) return if heating_description in ["Boiler and radiators, mains gas"]: # We can recommend roomstat programmer trvs - self.recommend_roomstat_programmer_trvs() + self.recommend_roomstat_programmer_trvs(description_suffix=description_suffix) # We can also recommend time and temperature zone controls - self.recommend_time_temperature_zone_controls() + self.recommend_time_temperature_zone_controls(description_suffix=description_suffix) return @@ -94,16 +97,22 @@ class HeatingControlRecommender: # We don't implement any other recommendations right now return - def recommend_high_heat_retention_controls(self): + def recommend_high_heat_retention_controls(self, description_prefix=""): """ When applicable, we recommend upgrading the heating controls to high heat retention controls. This is a specific type of control system that is designed to work with electric storage heaters. It is a more efficient control system than the standard controls that come with electric storage heaters. We can then consider the heating system itself + + If there is a description prefix, this means there is a dual heating system and so we need to add this to the + description + :return: """ new_description = "Controls for high heat retention storage heaters" + if description_prefix: + new_description = f"{description_prefix}, {new_description}" # We recommend upgrading to Celect type controls ending_config = MainheatControlAttributes(new_description).process() @@ -112,7 +121,10 @@ class HeatingControlRecommender: new_config=ending_config, old_config=self.property.main_heating_controls ) # This upgrade will only take the heating system to average energy efficiency - simulation_config["mainheatc_energy_eff_ending"] = "Good" + if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average"]: + simulation_config["mainheatc_energy_eff_ending"] = "Good" + else: + simulation_config["mainheatc_energy_eff_ending"] = self.property.data["mainheatc-energy-eff"] description_simulation = { "mainheatcont-description": new_description, @@ -131,7 +143,7 @@ class HeatingControlRecommender: # We don't implement any other recommendations right now return - def recommend_roomstat_programmer_trvs(self): + def recommend_roomstat_programmer_trvs(self, description_suffix=""): """ If the home has a boiler and radiators, mains gas, we start by identifying potential heating controls that could be upgraded, that would provide a practical impact. @@ -163,6 +175,8 @@ class HeatingControlRecommender: return new_controls_description = "Programmer, room thermostat and TRVS" + if description_suffix: + new_controls_description = f"{new_controls_description}, {description_suffix}" ending_config = MainheatControlAttributes(new_controls_description).process() # We use this to determine how we should be updating the config @@ -216,7 +230,7 @@ class HeatingControlRecommender: return - def recommend_time_temperature_zone_controls(self): + def recommend_time_temperature_zone_controls(self, description_suffix=""): """ If the home has a boiler, we can recommend time and temperature zone controls. This is a more advanced and more efficient control system than the standard controls that come with a boiler. However, it may come @@ -238,6 +252,8 @@ class HeatingControlRecommender: return new_controls_description = "Time and temperature zone control" + if description_suffix: + new_controls_description = f"{new_controls_description}, {description_suffix}" ending_config = MainheatControlAttributes(new_controls_description).process() @@ -260,8 +276,10 @@ class HeatingControlRecommender: number_heated_rooms=int(self.property.data["number-heated-rooms"]) ) - description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " - "temperature zone control)") + description = ( + "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " + "temperature zone control)" + ) already_installed = "heating_control" in self.property.already_installed if already_installed: diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index bb074407..23b9bf7d 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,5 +1,7 @@ from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE -from recommendations.recommendation_utils import check_simulation_difference, override_costs +from recommendations.recommendation_utils import ( + check_simulation_difference, override_costs, combine_recommendation_configs +) from backend.Property import Property from backend.app.plan.schemas import MEASURE_MAP from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes @@ -11,6 +13,37 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" + DUAL_HEATING_DESCRIPTIONS = { + "Boiler and radiators, mains gas, electric storage heaters": { + "hhr": { + "mainheating_description": "Boiler and radiators, mains gas, Electric storage heaters", + "recommendation_description": "Install high heat retention electric storage heaters alongside the " + "boiler. The current electric heaters may be retrofit with high heat " + "retention storage controls" + " however this is dependent on the existing system and may not be " + "possible.", + "controls_prefix": "current_controls" + }, + "boiler": { + "mainheating_description": "Boiler and radiators, mains gas, electric storage heaters", + "recommendation_description": "Upgrade the existing boiler to a new, more efficient condensing " + "boiler. ", + "controls_suffix": "Manual charge controls" + }, + # These are the heating types we need to produce a dual heating recommendation + "dual": { + "recommendation_description": "Upgrade both the existing boiler to a new condensing boiler and" + "upgrade storage heaters to high heat retention storage heaters.", + "types": [ + # type 1 + "boiler_upgrade", + # type 2 + "high_heat_retention_storage_heater", + ] + } + } + } + def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -20,10 +53,34 @@ class HeatingRecommender: self.has_electric_heating_description = self.property.main_heating["has_electric"] self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] + self.has_room_heaters = ( + self.property.main_heating["has_room_heaters"] or + self.property.main_heating["has_portable_electric_heaters"] + ) + self.has_boiler = self.property.main_heating["has_boiler"] + + self.dual_heating = self.identify_dual_heating() + + def identify_dual_heating(self): + # All heat systems are in here so we identify whether two of these are true + # MainHeatAttributes.HEAT_SYSTEMS + + n_trues = 0 + for heat_system in MainHeatAttributes.HEAT_SYSTEMS: + if self.property.main_heating[f"has_{heat_system.replace(' ', '_')}"]: + n_trues += 1 + + if n_trues > 2 or n_trues == 0: + raise Exception("Implement me") + if n_trues == 1: + return False + + return True def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ Check conditions if high heat retention storage is valid + If there's already an ASHP in place, we don't recommend HHR :return: """ @@ -32,8 +89,6 @@ class HeatingRecommender: hhr_suitable = no_mains or self.has_electric_heating_description - # If there's already an ASHP in place, we don't recommend HHR - return ( hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and ("high_heat_retention_storage_heater" in measures) @@ -47,7 +102,7 @@ class HeatingRecommender: # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls # If it's NOT a gas boiler, we'll potentially recommend a boiler - has_gas_boiler = self.property.main_heating["has_boiler"] and self.property.main_heating["has_mains_gas"] + has_gas_boiler = self.has_boiler and self.property.main_heating["has_mains_gas"] # 2) If the property doesn't have a heating system, but it has access to the mains gas no_heating_has_mains = self.property.main_heating["clean_description"] in [ @@ -55,21 +110,13 @@ class HeatingRecommender: ] and self.property.data["mains-gas-flag"] # The property is using portable heaters and has access to gas mains - has_room_heaters = ( - ( - self.property.main_heating["has_room_heaters"] or - self.property.main_heating["has_portable_electric_heaters"] - ) and - self.property.data["mains-gas-flag"] - ) + has_room_heaters = self.has_room_heaters and self.property.data["mains-gas-flag"] # We also check if the property has electric heating, but it has access to the mains gas electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"] portable_heaters_has_mains = ( - self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"] - and - self.property.data["mains-gas-flag"] + self.property.main_heating["has_portable_electric_heaters"] and self.property.data["mains-gas-flag"] ) # The next condition is if the home has a non-gas boiler, such as an oil boiler, with a mains gas connection @@ -95,6 +142,55 @@ class HeatingRecommender: return is_valid, has_gas_boiler + def recommend_dual_heating(self): + + if self.property.main_heating["clean_description"] not in self.DUAL_HEATING_DESCRIPTIONS: + return + + dual_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"]["types"] + + recommendation_system_types = list(set([x["system_type"] for x in self.heating_recommendations])) + + # We check if we have the required type + if not any([x in recommendation_system_types for x in dual_heating_description]): + return + + type_1_recommendations = [ + x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[0] + ] + type_2_recommendations = [ + x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[1] + ] + # we combine the two recommendations + combined_recommendations = [] + for rec in type_1_recommendations: + for rec2 in type_2_recommendations: + combined_rec = rec.copy() + # Update the description + combined_rec["description"] = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"]["recommendation_description"] + + # Combine simulation_config + # Make sure we end up with the best efficiecy values + combined_rec["simulation_config"] = combine_recommendation_configs( + rec["simulation_config"], rec2["simulation_config"] + ) + # Combine description_simulation + combined_rec["description_simulation"] = combine_recommendation_configs( + rec["description_simulation"], rec2["description_simulation"] + ) + + # Combine costs + for k in ["total", "subtotal", "vat", "labour_hours", "labour_days"]: + combined_rec[k] = rec[k] + rec2[k] + + combined_recommendations.append(combined_rec) + + self.heating_recommendations.extend(combined_recommendations) + def recommend(self, has_cavity_or_loft_recommendations, phase=0, measures=None): """ Produces heating recommendations @@ -144,14 +240,16 @@ class HeatingRecommender: # This indicates that the home previously did not have a boiler in place and so would require # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler system_change = not has_gas_boiler - exising_room_heaters = self.property.main_heating["clean_description"] in [ - "Room heaters, electric", "Room heaters, mains gas" - ] + exising_room_heaters = self.property.main_heating["has_room_heaters"] self.recommend_boiler_upgrades( phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters ) + # If we have dual heating and we allow for a combined recommendation, to upgrade both systems + if self.dual_heating: + self.recommend_dual_heating(phase=phase, measures=measures) + # We recommend air source heat pumps # Heat pumps are suitable for all property types: # https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/ @@ -424,7 +522,8 @@ class HeatingRecommender: description, phase, heating_controls_only, - system_change + system_change, + system_type ): """ Given a recommendation for heating controls, and a recommendation for the heating system, we combine the two @@ -439,6 +538,7 @@ class HeatingRecommender: :param system_change: Indicates if we are recommending a different type of heating system, compared to the current system. If we have a system change and we have a heat control recommendation, we only recommend both heating and controls together + :param system_type: The type of heating system we are recommending :return: """ @@ -494,7 +594,9 @@ class HeatingRecommender: "already_installed": already_installed, **total_costs, "simulation_config": recommendation_simulation_config, - "description_simulation": recommendation_description_simulation + "description_simulation": recommendation_description_simulation, + # We insert the heating system type here + "system_type": system_type } output.append(recommendation) @@ -572,7 +674,19 @@ class HeatingRecommender: # We only recommend Celect-type controls if the current heating system is not Celect-type controls if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc: - controls_recommender.recommend(heating_description="Electric storage heaters, radiators") + if self.dual_heating: + if self.DUAL_HEATING_DESCRIPTIONS[self.property.main_heating["clean_description"]]["hhr"][ + "controls_prefix" + ] == "current_controls": + description_prefix = self.property.main_heating_controls["clean_description"] + else: + raise NotImplementedError("Implement me") + else: + description_prefix = "" + + controls_recommender.recommend( + heating_description="Electric storage heaters, radiators", description_prefix=description_prefix + ) has_hhr = self.is_hhr_already_installed() # Conditions for not recommending electric storage heaters @@ -580,7 +694,13 @@ class HeatingRecommender: # No recommendation needed return - new_heating_description = "Electric storage heaters, radiators" + # We check if the property has dual heating in place with a boiler and storage heaters + if self.dual_heating: + new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["mainheating_description"] + else: + new_heating_description = "Electric storage heaters, radiators" # Set up artefacts, suitable for the simulation and regardless of controls heating_ending_config = MainHeatAttributes(new_heating_description).process() @@ -588,7 +708,10 @@ class HeatingRecommender: new_config=heating_ending_config, old_config=self.property.main_heating ) # This upgrade will only take the heating system to average energy efficiency - heating_simulation_config["mainheat_energy_eff_ending"] = "Average" + if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor"]: + heating_simulation_config["mainheat_energy_eff_ending"] = "Average" + else: + heating_simulation_config["mainheat_energy_eff_ending"] = self.property.data["mainheat-energy-eff"] # If the property is off-gas and has no heating system in place, the number of heated rooms will actually # be 0, so we use the number of rooms as the figure @@ -603,7 +726,13 @@ class HeatingRecommender: costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms ) - description = "Install high heat retention electric storage heaters." + if self.dual_heating: + description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["recommendation_description"] + + else: + description = "Install high heat retention electric storage heaters." # We check the existing heating system and controls if ( @@ -627,7 +756,8 @@ class HeatingRecommender: description=description, phase=phase, heating_controls_only=heating_controls_only, - system_change=system_change + system_change=system_change, + system_type="high_heat_retention_storage_heater" ) if _return: return recommendations @@ -721,11 +851,26 @@ class HeatingRecommender: num_heated_rooms=self.property.data["number-heated-rooms"], ) - description = "Upgrade to a new condensing boiler" + if self.dual_heating: + description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["recommendation_description"] + else: + description = "Upgrade to a new condensing boiler" + + new_heating_eff = ( + "Good" if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] + else self.property.data["mainheat-energy-eff"] + ) + + new_hotwater_eff = ( + "Good" if self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] + else self.property.data["hot-water-energy-eff"] + ) simulation_config = { - "mainheat_energy_eff_ending": "Good", - "hot_water_energy_eff_ending": "Good" + "mainheat_energy_eff_ending": new_heating_eff, + "hot_water_energy_eff_ending": new_hotwater_eff } description_simulation = { @@ -736,7 +881,13 @@ class HeatingRecommender: if system_change: # Installation of a boiler improves the hot water system so we need to reflect this in # the outcome of the recommendation - new_heating_description = "Boiler and radiators, mains gas" + if self.dual_heating: + new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["mainheating_description"] + else: + new_heating_description = "Boiler and radiators, mains gas" + new_hotwater_description = "From main system" new_fuel_description = "mains gas (not community)" @@ -794,13 +945,23 @@ class HeatingRecommender: "already_installed": already_installed, "simulation_config": simulation_config, "description_simulation": description_simulation, - **boiler_costs + **boiler_costs, + "system_type": "boiler_upgrade", } # We recommend the heating controls # If the property did not previously have a boiler, we combine controls_recommender = HeatingControlRecommender(self.property) - controls_recommender.recommend(heating_description="Boiler and radiators, mains gas") + if self.dual_heating: + description_suffix = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["controls_suffix"] + else: + description_suffix = "" + controls_recommender.recommend( + heating_description="Boiler and radiators, mains gas", + description_suffix=description_suffix + ) # We may have 2 recommendations from the heating controls if not controls_recommender.recommendation and not boiler_recommendation: @@ -822,7 +983,8 @@ class HeatingRecommender: description=boiler_recommendation["description"], phase=recommendation_phase, heating_controls_only=False, - system_change=True + system_change=True, + system_type="boiler_upgrade" ) combined_recommendations.extend(combined_recommendation) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index ce32e061..883a387b 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -800,3 +800,44 @@ def override_costs(costs): costs[k] = 0 return costs + + +def combine_recommendation_configs(recommendation_config1, recommendation_config2): + """ + Given two simulation configs, this function will combine them into one + :param recommendation_config1: + :param recommendation_config2: + :return: + """ + # Efficiency values - keys which contain _energy_eff_ending + eff_1 = { + k: v for k, v in recommendation_config1.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + } + eff_2 = { + k: v for k, v in recommendation_config2.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + } + + # We combine the simulation configs + combined = { + **recommendation_config1, + **recommendation_config2 + } + + # Find overlapping keys + overlapping_keys = set(eff_1.keys()).intersection(set(eff_2.keys())) + if overlapping_keys: + # We make sure we take the best value - map efficiency values to numbers + numerical_embedding = { + "Very poor": 1, + "Poor": 2, + "Average": 3, + "Good": 4, + "Very good": 5, + } + for key in overlapping_keys: + if numerical_embedding[eff_1[key]] >= numerical_embedding[eff_2[key]]: + combined[key] = eff_1[key] + else: + combined[key] = eff_2[key] + + return combined diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 51d0636e..821e79c6 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1066,6 +1066,99 @@ testing_examples = [ "heating_recommendation_descriptions": [], "heating_controls_recommendation_descriptions": [], "notes": "This property already has an ashp. We don't recommend any heating upgrades" + }, + { + "epc": { + 'lmk-key': '1dd9aa80d6e5bae3e0e4892d9ed1a83b53f3af848568f4a928c9f7a63d8825ea', + 'address1': '49 Ridgeway Road', 'address2': 'Wordsley', 'address3': None, 'postcode': 'DY8 5UD', + 'building-reference-number': 10003464876, 'current-energy-rating': 'F', 'potential-energy-rating': 'D', + 'current-energy-efficiency': 35, 'potential-energy-efficiency': 64, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-11-17', 'local-authority': 'E08000027', + 'constituency': 'E14000672', 'county': None, 'lodgement-date': '2022-10-10', 'transaction-type': 'rental', + 'environment-impact-current': 41, 'environment-impact-potential': 67, 'energy-consumption-current': 401, + 'energy-consumption-potential': 207, 'co2-emissions-current': 6.1, 'co2-emiss-curr-per-floor-area': 69, + 'co2-emissions-potential': 3.2, 'lighting-cost-current': 61, 'lighting-cost-potential': 61, + 'heating-cost-current': 1488, 'heating-cost-potential': 1015, 'hot-water-cost-current': 114, + 'hot-water-cost-potential': 77, 'total-floor-area': 89.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 91, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Good', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', 'windows-env-eff': 'Good', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, mains gas, Electric storage heaters', + 'mainheat-energy-eff': 'Good', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 91% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': None, + 'unheated-corridor-length': None, 'floor-height': 2.53, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '49 Ridgeway Road, Wordsley', + 'local-authority-label': 'Dudley', 'constituency-label': 'Dudley South', 'posttown': 'Stourbridge', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2022-10-10 16:41:36', + 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, + 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' + 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' + 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" + "we recommend ASHP and HHR" + }, + { + "epc": { + 'lmk-key': '1dd9aa80d6e5bae3e0e4892d9ed1a83b53f3af848568f4a928c9f7a63d8825ea', + 'address1': '49 Ridgeway Road', 'address2': 'Wordsley', 'address3': None, 'postcode': 'DY8 5UD', + 'building-reference-number': 10003464876, 'current-energy-rating': 'F', 'potential-energy-rating': 'D', + 'current-energy-efficiency': 35, 'potential-energy-efficiency': 64, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-11-17', 'local-authority': 'E08000027', + 'constituency': 'E14000672', 'county': None, 'lodgement-date': '2022-10-10', 'transaction-type': 'rental', + 'environment-impact-current': 41, 'environment-impact-potential': 67, 'energy-consumption-current': 401, + 'energy-consumption-potential': 207, 'co2-emissions-current': 6.1, 'co2-emiss-curr-per-floor-area': 69, + 'co2-emissions-potential': 3.2, 'lighting-cost-current': 61, 'lighting-cost-potential': 61, + 'heating-cost-current': 1488, 'heating-cost-potential': 1015, 'hot-water-cost-current': 114, + 'hot-water-cost-potential': 77, 'total-floor-area': 89.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 91, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Good', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', 'windows-env-eff': 'Good', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, mains gas, Electric storage heaters', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 91% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': None, + 'unheated-corridor-length': None, 'floor-height': 2.53, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '49 Ridgeway Road, Wordsley', + 'local-authority-label': 'Dudley', 'constituency-label': 'Dudley South', 'posttown': 'Stourbridge', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2022-10-10 16:41:36', + 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, + 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property is a modified version of the previous dual heating property, where we lower the" + "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" + "storage heaters" } ] @@ -1125,6 +1218,8 @@ completed_descriptions = [ "Boiler and radiators, smokeless fuel", "Boiler and radiators, wood pellets", "Room heaters, dual fuel (mineral and wood)", + "Air source heat pump, radiators, electric", + "Portable electric heaters assumed for most rooms, Room heaters, electric", ] portfolio = pd.read_excel( @@ -1139,7 +1234,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Air source heat pump, radiators, electric") + (portfolio["mainheat-description"] == "Boiler and radiators, mains gas, Electric storage heaters") ].sample(1) eg = eg.squeeze().to_dict() print(eg) diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 35373729..b80780d9 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -53,6 +53,8 @@ class TestHeatingRecommendations: we retrieve alongside them :return: """ + if test_case["epc"]["uprn"] == 90041166: + raise Exception("Finish the second test case with this uprn") epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} From 3b325395b9fd5ab61fe76c4af07bb29806ba64f4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 09:28:01 +0100 Subject: [PATCH 16/59] added dual heating recommendations --- recommendations/HeatingRecommender.py | 4 ++-- .../tests/test_data/heating_recommendations_data.py | 9 +++++++++ recommendations/tests/test_heating_recommendations.py | 2 -- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 23b9bf7d..db83508b 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -33,7 +33,7 @@ class HeatingRecommender: # These are the heating types we need to produce a dual heating recommendation "dual": { "recommendation_description": "Upgrade both the existing boiler to a new condensing boiler and" - "upgrade storage heaters to high heat retention storage heaters.", + " upgrade storage heaters to high heat retention storage heaters.", "types": [ # type 1 "boiler_upgrade", @@ -248,7 +248,7 @@ class HeatingRecommender: # If we have dual heating and we allow for a combined recommendation, to upgrade both systems if self.dual_heating: - self.recommend_dual_heating(phase=phase, measures=measures) + self.recommend_dual_heating() # We recommend air source heat pumps # Heat pumps are suitable for all property types: diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 821e79c6..76b8b218 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1154,6 +1154,15 @@ testing_examples = [ 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Upgrade the existing boiler to a new, more efficient condensing boiler. ', + 'Upgrade both the existing boiler to a new condensing boiler and upgrade storage heaters to high heat ' + 'retention storage heaters.', + 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' + 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' + 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], "notes": "This property is a modified version of the previous dual heating property, where we lower the" diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index b80780d9..35373729 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -53,8 +53,6 @@ class TestHeatingRecommendations: we retrieve alongside them :return: """ - if test_case["epc"]["uprn"] == 90041166: - raise Exception("Finish the second test case with this uprn") epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} From 59864bc8e8d3b64b1a091a7a2db4a22889238b00 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 10:05:56 +0100 Subject: [PATCH 17/59] added new room heatings unit tests --- .../test_data/heating_recommendations_data.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 76b8b218..77126c7b 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1168,6 +1168,55 @@ testing_examples = [ "notes": "This property is a modified version of the previous dual heating property, where we lower the" "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" "storage heaters" + }, + { + "epc": { + 'lmk-key': '670443469402019100713595382910638', 'address1': '2 Crabs Castle', 'address2': 'Pontrilas', + 'address3': None, 'postcode': 'HR2 0BN', 'building-reference-number': 6466069868, + 'current-energy-rating': 'G', 'potential-energy-rating': 'C', 'current-energy-efficiency': 20, + 'potential-energy-efficiency': 80, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2019-10-07', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2019-10-07', 'transaction-type': 'rental (social)', 'environment-impact-current': 1, + 'environment-impact-potential': 103, 'energy-consumption-current': 618, 'energy-consumption-potential': 110, + 'co2-emissions-current': 15.0, 'co2-emiss-curr-per-floor-area': 206, 'co2-emissions-potential': -0.3, + 'lighting-cost-current': 125, 'lighting-cost-potential': 63, 'heating-cost-current': 1371, + 'heating-cost-potential': 473, 'hot-water-cost-current': 524, 'hot-water-cost-potential': 101, + 'total-floor-area': 73.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, + 'low-energy-lighting': 0, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, off-peak', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': 'NO DATA!', 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', + 'windows-env-eff': 'Good', 'walls-description': 'Cavity wall, as built, no insulation (assumed)', + 'walls-energy-eff': 'Poor', 'walls-env-eff': 'Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 150 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Room heaters, anthracite', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Very Poor', 'mainheatcont-description': 'No thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'HEREFORD', + 'construction-age-band': 'England and Wales: 1930-1949', 'lodgement-datetime': '2019-10-07 13:59:53', + 'tenure': 'rental (social)', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, + 'uprn': 10009574286, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has anthracite heating without mains. " + "We recommend ASHP and HHR, but no gas condensing boiler" } ] @@ -1229,6 +1278,8 @@ completed_descriptions = [ "Room heaters, dual fuel (mineral and wood)", "Air source heat pump, radiators, electric", "Portable electric heaters assumed for most rooms, Room heaters, electric", + "Boiler and radiators, mains gas, Electric storage heaters", + "Room heaters, anthracite", ] portfolio = pd.read_excel( @@ -1243,7 +1294,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, mains gas, Electric storage heaters") + (portfolio["mainheat-description"] == "Room heaters, anthracite") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 185406026666c12b37a29488edb5b47e9c3d9569 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 10:27:51 +0100 Subject: [PATCH 18/59] added additional heating test case --- recommendations/HeatingRecommender.py | 5 +- .../test_data/heating_recommendations_data.py | 52 ++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index db83508b..e4107205 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -87,7 +87,10 @@ class HeatingRecommender: # We can also recommend hhr if the property doesn't have a mains has connection no_mains = not self.property.data["mains-gas-flag"] - hhr_suitable = no_mains or self.has_electric_heating_description + # If the property already has room heaters then we recommend HHR as an option since the home already has + # a variation of room heaters + + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 77126c7b..58f3fad4 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1217,6 +1217,55 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has anthracite heating without mains. " "We recommend ASHP and HHR, but no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '298ecac47f69461257e582e9dd78cb4c8bbb8c05bee1af1530f052134ecd5044', 'address1': '9 HENDON RISE', + 'address2': None, 'address3': None, 'postcode': 'NG3 3AN', 'building-reference-number': 10001202381, + 'current-energy-rating': 'G', 'potential-energy-rating': 'D', 'current-energy-efficiency': 12, + 'potential-energy-efficiency': 66, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2021-03-31', 'local-authority': 'E06000018', 'constituency': 'E14000865', + 'county': None, + 'lodgement-date': '2021-05-27', 'transaction-type': 'rental', 'environment-impact-current': 15, + 'environment-impact-potential': 59, 'energy-consumption-current': 630, 'energy-consumption-potential': 187, + 'co2-emissions-current': 11.0, 'co2-emiss-curr-per-floor-area': 122, 'co2-emissions-potential': 3.6, + 'lighting-cost-current': 99, 'lighting-cost-potential': 99, 'heating-cost-current': 2107, + 'heating-cost-potential': 860, 'hot-water-cost-current': 458, 'hot-water-cost-potential': 73, + 'total-floor-area': 91.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': None, + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 5, 'number-heated-rooms': 2, + 'low-energy-lighting': 63, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Portable electric heaters (assumed)', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', + 'mainheat-description': 'Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 63% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.79, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '9 HENDON RISE', 'local-authority-label': 'Nottingham', 'constituency-label': 'Nottingham East', + 'posttown': 'NOTTINGHAM', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2021-05-27 13:52:20', 'tenure': 'Rented (private)', + 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 100031556691, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has room heaters with two different fuel sources" } ] @@ -1280,6 +1329,7 @@ completed_descriptions = [ "Portable electric heaters assumed for most rooms, Room heaters, electric", "Boiler and radiators, mains gas, Electric storage heaters", "Room heaters, anthracite", + "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)" ] portfolio = pd.read_excel( @@ -1294,7 +1344,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, anthracite") + (portfolio["mainheat-description"] == "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 5243e64e4128656390e4c7ffad862f4e6c319b9e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 11:08:57 +0100 Subject: [PATCH 19/59] Added additional heating unit test --- recommendations/HeatingRecommender.py | 4 +- .../test_data/heating_recommendations_data.py | 105 +++++++++++++++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index e4107205..e40c1736 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -51,7 +51,9 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = self.property.main_heating["has_electric"] + self.has_electric_heating_description = ( + self.property.main_heating["has_electric"] or self.property.main_heating["has_electricaire"] + ) self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] self.has_room_heaters = ( self.property.main_heating["has_room_heaters"] or diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 58f3fad4..fea53e2b 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1265,7 +1265,105 @@ testing_examples = [ 'Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], - "notes": "This property has room heaters with two different fuel sources" + "notes": "This property has room heaters with two different fuel sources, so we recommend HHR, ASHP, and a " + "boiler upgrade" + }, + { + "epc": { + 'lmk-key': '717779210932011102812033766268396', 'address1': '28, Overdale Road', 'address2': None, + 'address3': None, 'postcode': 'CV5 8AL', 'building-reference-number': 1616392968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 28, + 'potential-energy-efficiency': 40, 'property-type': 'Flat', 'built-form': 'NO DATA!', + 'inspection-date': '2011-10-27', 'local-authority': 'E08000026', 'constituency': 'E14000650', + 'county': None, + 'lodgement-date': '2011-10-28', 'transaction-type': 'marketed sale', 'environment-impact-current': 37, + 'environment-impact-potential': 47, 'energy-consumption-current': 544, 'energy-consumption-potential': 431, + 'co2-emissions-current': 5.1, 'co2-emiss-curr-per-floor-area': 96, 'co2-emissions-potential': 4.0, + 'lighting-cost-current': 54, 'lighting-cost-potential': 31, 'heating-cost-current': 576, + 'heating-cost-potential': 711, 'hot-water-cost-current': 598, 'hot-water-cost-potential': 232, + 'total-floor-area': 52.93, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': '1st', + 'flat-top-storey': 'N', 'flat-storey-count': None, 'main-heating-controls': 2703.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 3, + 'low-energy-lighting': 25, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': '(other premises below)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'None', 'roof-description': '(another dwelling above)', + 'roof-energy-eff': None, 'roof-env-eff': None, 'mainheat-description': 'Electric underfloor heating', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'Room thermostat only', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 25% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'unheated corridor', 'unheated-corridor-length': 4.44, + 'floor-height': 2.28, 'photo-supply': 0.0, 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '28, Overdale Road', 'local-authority-label': 'Coventry', + 'constituency-label': 'Coventry North West', 'posttown': 'COVENTRY', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2011-10-28 12:03:37', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': 2.0, + 'uprn': 100070685908, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property is a flag, without mains gas connection. Currently has underfloor electric heating" + "so we recommend HHR" + }, + { + "epc": { + 'lmk-key': '800099229502014050214593392940598', 'address1': '16, Woodside', 'address2': None, + 'address3': None, 'postcode': 'WV11 2PT', 'building-reference-number': 4556129968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 21, + 'potential-energy-efficiency': 39, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2014-05-01', 'local-authority': 'E08000031', 'constituency': 'E14001049', + 'county': None, + 'lodgement-date': '2014-05-02', 'transaction-type': 'none of the above', 'environment-impact-current': 32, + 'environment-impact-potential': 46, 'energy-consumption-current': 479, 'energy-consumption-potential': 340, + 'co2-emissions-current': 8.1, 'co2-emiss-curr-per-floor-area': 85, 'co2-emissions-potential': 5.7, + 'lighting-cost-current': 111, 'lighting-cost-potential': 55, 'heating-cost-current': 2021, + 'heating-cost-potential': 1655, 'hot-water-cost-current': 408, 'hot-water-cost-potential': 189, + 'total-floor-area': 96.0, 'energy-tariff': 'Unknown', 'mains-gas-flag': 'Y', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2504.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'System built, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Warm air, Electricaire', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Very Poor', 'mainheatcont-description': 'Programmer and room thermostat', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', + 'address': '16, Woodside', 'local-authority-label': 'Wolverhampton', + 'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON', + 'construction-age-band': 'England and Wales: 1967-1975', 'lodgement-datetime': '2014-05-02 14:59:33', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 20.0, 'low-energy-fixed-light-count': 0.0, + 'uprn': 100071209105, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" + "connection so we recommend a gas condensing boiler" } ] @@ -1329,7 +1427,8 @@ completed_descriptions = [ "Portable electric heaters assumed for most rooms, Room heaters, electric", "Boiler and radiators, mains gas, Electric storage heaters", "Room heaters, anthracite", - "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)" + "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", + "Electric underfloor heating", ] portfolio = pd.read_excel( @@ -1344,7 +1443,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)") + (portfolio["mainheat-description"] == "Warm air, Electricaire") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 53e68f8d76b3f8146166ba575c7507908e3b1643 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 15:26:38 +0100 Subject: [PATCH 20/59] added tests for final property --- etl/sfr/midlands_portfolio_est_funding.py | 100 ++++++--- recommendations/HeatingRecommender.py | 19 +- .../test_data/heating_recommendations_data.py | 209 +++++++++++------- 3 files changed, 218 insertions(+), 110 deletions(-) diff --git a/etl/sfr/midlands_portfolio_est_funding.py b/etl/sfr/midlands_portfolio_est_funding.py index 09102cfb..017fd223 100644 --- a/etl/sfr/midlands_portfolio_est_funding.py +++ b/etl/sfr/midlands_portfolio_est_funding.py @@ -57,10 +57,18 @@ def app(): ['none', "below average"] ) + epc_data["needs_solid_wall"] = (epc_data["is_solid_brick"] | epc_data["is_system_built"]) & epc_data[ + "insulation_thickness_wall"].isin(['none', "below average"]) + + epc_data["could_take_solar"] = (epc_data["is_flat"] | epc_data["is_pitched"]) + loft_insulation_per_m2 = 16.07 flat_roof_insulation_per_m2 = 195 cwi_per_m2 = 14.21 + ewi_per_m2 = 200 gbis_abs = 30 + eco4_abs = 24 + solar_pv_cost = 4009 # We assume the work will take the home from a high D to a low D def get_abs(floor_area): @@ -75,6 +83,19 @@ def app(): return 350.1 + # We assume the work will take the home from a high E to a high C + def get_eco4_abs(floor_area): + if floor_area <= 72: + return 596.6 + + if floor_area <= 97: + return 650.2 + + if floor_area <= 199: + return 755.8 + + return 1347.1 + estimated_costs = [] for _, home in epc_data.iterrows(): to_append = { @@ -89,30 +110,59 @@ def app(): n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"]) floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5 - # Check if it needs the walls done - if home["needs_cavity_done"]: - # We estimate the amount of insulation required - est_perimeter = estimate_perimeter( - floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, - num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors - ) + # We estimate the amount of insulation required + est_perimeter = estimate_perimeter( + floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, + num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors + ) - insulation_needed = estimate_external_wall_area( - num_floors=n_floors, - floor_height=floor_height, - perimeter=est_perimeter, - built_form=home["BUILT_FORM"], - ) - cost_of_insulation = insulation_needed * cwi_per_m2 + insulation_needed = estimate_external_wall_area( + num_floors=n_floors, + floor_height=floor_height, + perimeter=est_perimeter, + built_form=home["BUILT_FORM"], + ) - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + # At the very least we'll need solid wall + solar + if home["needs_solid_wall"] and home["could_take_solar"]: + measure = "EWI + Solar" + + total_cost = insulation_needed * ewi_per_m2 + solar_pv_cost + + eco4_project_abs = get_eco4_abs(home["TOTAL_FLOOR_AREA"]) + eco4_available_funding = eco4_project_abs * eco4_abs + + cost_of_work_after_funding = total_cost - eco4_available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "eco4", + "available_funding": eco4_available_funding, + "measure": measure, + "project_abs": eco4_project_abs, + "cost_of_work": total_cost, + "cost_of_work_after_funding": cost_of_work_after_funding, + } + + estimated_costs.append(to_append) + continue + + # Check if it needs the walls done + if home["needs_cavity_done"]: + cost_of_insulation = insulation_needed * cwi_per_m2 + + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding + + to_append = { + **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": "Cavity Wall Insulation", - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -130,14 +180,17 @@ def app(): roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors cost_of_insulation = roof_area * flat_roof_insulation_per_m2 - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": measure, - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -145,13 +198,10 @@ def app(): estimated_costs = pd.DataFrame(estimated_costs) - estimated_costs.groupby("measure")["available_funding"].mean() - estimated_costs["measure"].value_counts() - estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv") - epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") + # epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( + # "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") n_properties_for_ashp = epc_data[ (epc_data["PROPERTY_TYPE"] == "House") & diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index e40c1736..fea2d8db 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -91,7 +91,7 @@ class HeatingRecommender: # If the property already has room heaters then we recommend HHR as an option since the home already has # a variation of room heaters - + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( @@ -130,6 +130,13 @@ class HeatingRecommender: not self.property.main_heating["has_mains_gas"] and self.property.data["mains-gas-flag"] ) + # Additionally, if the property has a gas connection, is using gas heating but doesn't have a boiler, + # we recommend a boiler + non_boiler_gas_heating = ( + self.property.data["mains-gas-flag"] and + self.property.main_heating["has_mains_gas"] and + not self.property.main_heating["has_boiler"] + ) is_valid = ( ( @@ -138,7 +145,8 @@ class HeatingRecommender: electic_heating_has_mains or has_room_heaters or portable_heaters_has_mains or - non_gas_boiler + non_gas_boiler or + non_boiler_gas_heating ) and (not ashp_only_heating_recommendation) and ("boiler_upgrade" in measures) and @@ -842,12 +850,13 @@ class HeatingRecommender: has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] - has_inefficient_mains_water = ( - self.property.hotwater["clean_description"] in ["From main system"] and + # We check if there's a mains connection and the hot water is inefficient, as this will improve with a boiler + has_inefficient_water = ( + self.property.data["mains-gas-flag"] and self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] ) - if has_inefficient_space_heating or has_inefficient_mains_water: + if has_inefficient_space_heating or has_inefficient_water: boiler_size = self.estimate_boiler_size( property_type=self.property.data["property-type"], built_form=self.property.data["built-form"], diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index fea53e2b..e6225299 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1364,86 +1364,135 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" "connection so we recommend a gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '272170070262009042917361440218801', 'address1': '52, Chiswick Walk', 'address2': None, + 'address3': None, 'postcode': 'B37 6TA', 'building-reference-number': 479790668, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 31, + 'potential-energy-efficiency': 50, 'property-type': 'Flat', 'built-form': 'End-Terrace', + 'inspection-date': '2009-04-29', 'local-authority': 'E08000029', 'constituency': 'E14000812', + 'county': None, + 'lodgement-date': '2009-04-29', 'transaction-type': 'marketed sale', 'environment-impact-current': 37, + 'environment-impact-potential': 42, 'energy-consumption-current': 548, 'energy-consumption-potential': 459, + 'co2-emissions-current': 5.8, 'co2-emiss-curr-per-floor-area': 89, 'co2-emissions-potential': 5.0, + 'lighting-cost-current': 60, 'lighting-cost-potential': 30, 'heating-cost-current': 751, + 'heating-cost-potential': 601, 'hot-water-cost-current': 239, 'hot-water-cost-potential': 129, + 'total-floor-area': 65.04, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': '1st', + 'flat-top-storey': 'Y', 'flat-storey-count': 1.0, 'main-heating-controls': 2504.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 2, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'To external air, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', + 'windows-env-eff': 'Good', 'walls-description': 'System built, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Portable electric heaters', + 'roof-description': 'Pitched, limited insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Warm air, mains gas', 'mainheat-energy-eff': 'Good', + 'mainheat-env-eff': 'Good', 'mainheatcont-description': 'Programmer and room thermostat', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'unheated corridor', 'unheated-corridor-length': 5.63, + 'floor-height': 2.32, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '52, Chiswick Walk', 'local-authority-label': 'Solihull', + 'constituency-label': 'Meriden', 'posttown': 'BIRMINGHAM', + 'construction-age-band': 'England and Wales: 1967-1975', 'lodgement-datetime': '2009-04-29 17:36:14', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, + 'uprn': 100070955137, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has warm air mains gas heating, so we recommend a gas condensing boiler" } ] -import random -from pathlib import Path -import inspect -import pandas as pd - -# this can be used to get example data to build the test cases -src_file_path = inspect.getfile(lambda: None) -EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" -epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] -directory = random.sample(epc_directories, 1)[0] -data = pd.read_csv(directory / "certificates.csv", low_memory=False) -# Rename the columns to the same format as the api returns -data.columns = [c.replace("_", "-").lower() for c in data.columns] -data["floor-height"] = data["floor-height"].fillna(2.45) - -used_examples = pd.DataFrame( - [ - { - "mainheat-description": x["epc"]["mainheat-description"], - "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], - "property-type": x["epc"]["property-type"], - "built-form": x["epc"]["built-form"], - "used": True - } for x in testing_examples - ] -) - -data = data.merge( - used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] -) -data = data[pd.isnull(data["used"])].drop(columns=["used"]) - -eg = data.sample(1).to_dict("records")[0] -print(eg["mainheat-description"]) -print(eg["mainheat-energy-eff"]) -print(eg["property-type"]) -print(eg["built-form"]) -print(eg["mainheatcont-description"]) - -### We also use the Midlands EPC F/G portfolio to get examples to create tests - -completed_descriptions = [ - "Portable electric heaters assumed for most rooms", - "Boiler and radiators, oil", - "Boiler and radiators, mains gas", - "Room heaters, mains gas", - "No system present: electric heaters assumed", - "Room heaters, electric", - "Electric storage heaters", - "Boiler and radiators, LPG", - "Boiler and radiators, electric", - "Boiler and radiators, dual fuel (mineral and wood)", - "Boiler and radiators, coal", - "Boiler and radiators, smokeless fuel", - "Boiler and radiators, wood pellets", - "Room heaters, dual fuel (mineral and wood)", - "Air source heat pump, radiators, electric", - "Portable electric heaters assumed for most rooms, Room heaters, electric", - "Boiler and radiators, mains gas, Electric storage heaters", - "Room heaters, anthracite", - "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", - "Electric underfloor heating", -] - -portfolio = pd.read_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" -) -portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] -portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -portfolio['sheating-energy-eff'] = None -portfolio['sheating-env-eff'] = None -portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) - -print(portfolio["mainheat-description"].value_counts()) - -eg = portfolio[ - (portfolio["mainheat-description"] == "Warm air, Electricaire") -].sample(1) -eg = eg.squeeze().to_dict() -print(eg) +# import random +# from pathlib import Path +# import inspect +# import pandas as pd +# +# # this can be used to get example data to build the test cases +# src_file_path = inspect.getfile(lambda: None) +# EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" +# epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] +# directory = random.sample(epc_directories, 1)[0] +# data = pd.read_csv(directory / "certificates.csv", low_memory=False) +# # Rename the columns to the same format as the api returns +# data.columns = [c.replace("_", "-").lower() for c in data.columns] +# data["floor-height"] = data["floor-height"].fillna(2.45) +# +# used_examples = pd.DataFrame( +# [ +# { +# "mainheat-description": x["epc"]["mainheat-description"], +# "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], +# "property-type": x["epc"]["property-type"], +# "built-form": x["epc"]["built-form"], +# "used": True +# } for x in testing_examples +# ] +# ) +# +# data = data.merge( +# used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] +# ) +# data = data[pd.isnull(data["used"])].drop(columns=["used"]) +# +# eg = data.sample(1).to_dict("records")[0] +# print(eg["mainheat-description"]) +# print(eg["mainheat-energy-eff"]) +# print(eg["property-type"]) +# print(eg["built-form"]) +# print(eg["mainheatcont-description"]) +# +# ### We also use the Midlands EPC F/G portfolio to get examples to create tests +# +# completed_descriptions = [ +# "Portable electric heaters assumed for most rooms", +# "Boiler and radiators, oil", +# "Boiler and radiators, mains gas", +# "Room heaters, mains gas", +# "No system present: electric heaters assumed", +# "Room heaters, electric", +# "Electric storage heaters", +# "Boiler and radiators, LPG", +# "Boiler and radiators, electric", +# "Boiler and radiators, dual fuel (mineral and wood)", +# "Boiler and radiators, coal", +# "Boiler and radiators, smokeless fuel", +# "Boiler and radiators, wood pellets", +# "Room heaters, dual fuel (mineral and wood)", +# "Air source heat pump, radiators, electric", +# "Portable electric heaters assumed for most rooms, Room heaters, electric", +# "Boiler and radiators, mains gas, Electric storage heaters", +# "Room heaters, anthracite", +# "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", +# "Electric underfloor heating", +# "Warm air, Electricaire" +# ] +# +# portfolio = pd.read_excel( +# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" +# ) +# portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +# portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +# portfolio['sheating-energy-eff'] = None +# portfolio['sheating-env-eff'] = None +# portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) +# +# print(portfolio["mainheat-description"].value_counts()) +# +# eg = portfolio[ +# (portfolio["mainheat-description"] == "Warm air, mains gas") +# ].sample(1) +# eg = eg.squeeze().to_dict() +# print(eg) From b34f1faca0e10fbabd00a01b07af6909391dbf37 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 16:28:07 +0100 Subject: [PATCH 21/59] adding new costs to backend --- etl/costs/app.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/etl/costs/app.py b/etl/costs/app.py index 59852cc5..85c2410e 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -11,7 +11,7 @@ import inspect src_file_path = inspect.getfile(lambda: None) -DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240626 Hestia Materials.xlsx" +DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240917 Hestia Materials.xlsx" # Environment file is at the same level as this file ENV_FILE = Path(src_file_path).parent / "etl" / "costs" / ".env" dotenv.load_dotenv(ENV_FILE) @@ -46,6 +46,17 @@ def push_costs_to_db(engine, costs_df): session.commit() +def set_current_costs_inactive(engine): + """ + Set all current costs to inactive in the database. + + :param engine: The SQLAlchemy engine connected to your database. + """ + with Session(engine) as session: + session.query(Material).update({Material.is_active: False}) + session.commit() + + def app(): """ This application uploads the cost data to our database @@ -108,6 +119,11 @@ def app(): costs[col] = costs[col].fillna(0) # Push the costs to the database + # Since this is just uploading all of the new costs to the database, we make all of the current costs inactive + print("Setting all current costs to inactive") + set_current_costs_inactive(db_engine) + + print("Pushing costs to db") push_costs_to_db(db_engine, costs) From 335823d3af3e34db42624b00d46353457282eb25 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 10:54:52 +0100 Subject: [PATCH 22/59] fixing incorrect descriptions and removing test expecations for homes that shouldn't have an ashp --- recommendations/HeatingRecommender.py | 39 +++++++++++-- .../test_data/heating_recommendations_data.py | 58 ++++++++----------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index fea2d8db..3daf0268 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -41,6 +41,20 @@ class HeatingRecommender: "high_heat_retention_storage_heater", ] } + }, + "Portable electric heaters assumed for most rooms, room heaters, electric": { + "hhr": { + "mainheating_description": "Electric storage heaters, radiators", + "recommendation_description": "Install high heat retention electric storage heaters.", + "controls_prefix": "" + }, + "boiler": { + "mainheating_description": "Boiler and radiators, mains gas", + "recommendation_description": "Upgrade to a new condensing boiler.", + "controls_suffix": "" + }, + # These are the heating types we need to produce a dual heating recommendation + "dual": None } } @@ -91,7 +105,7 @@ class HeatingRecommender: # If the property already has room heaters then we recommend HHR as an option since the home already has # a variation of room heaters - + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( @@ -160,6 +174,12 @@ class HeatingRecommender: if self.property.main_heating["clean_description"] not in self.DUAL_HEATING_DESCRIPTIONS: return + # if we have set dual to None, we do not produce a dual heating recommendation + if self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"] is None: + return + dual_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ self.property.main_heating["clean_description"] ]["dual"]["types"] @@ -688,10 +708,15 @@ class HeatingRecommender: # We only recommend Celect-type controls if the current heating system is not Celect-type controls if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc: if self.dual_heating: - if self.DUAL_HEATING_DESCRIPTIONS[self.property.main_heating["clean_description"]]["hhr"][ - "controls_prefix" - ] == "current_controls": + + controls_prefix = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["controls_prefix"] + + if controls_prefix == "current_controls": description_prefix = self.property.main_heating_controls["clean_description"] + elif controls_prefix == "": + description_prefix = "" else: raise NotImplementedError("Implement me") else: @@ -735,6 +760,10 @@ class HeatingRecommender: self.property.number_of_rooms ) ) + # To be conservative, we adjust if we still have 1 room + if (number_heated_rooms == 1) and (self.property.number_of_rooms > 2): + number_heated_rooms = self.property.number_of_rooms - 1 + # Upgrade to electric storage heaters costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms @@ -870,7 +899,7 @@ class HeatingRecommender: self.property.main_heating["clean_description"] ]["boiler"]["recommendation_description"] else: - description = "Upgrade to a new condensing boiler" + description = "Upgrade to a new condensing boiler." new_heating_eff = ( "Good" if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index e6225299..e073ac99 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -94,7 +94,7 @@ testing_examples = [ 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - "Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention " + "Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention " "Storage Heater Controls" ], "heating_controls_recommendation_descriptions": [], @@ -198,7 +198,7 @@ testing_examples = [ 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler' + 'Upgrade to a new condensing boiler.' ], "heating_controls_recommendation_descriptions": [ 'Upgrade heating controls to Room thermostat, programmer and TRVs', @@ -251,17 +251,14 @@ testing_examples = [ 'uprn': 100021936225.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', ], "heating_controls_recommendation_descriptions": [ 'upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' ], - "notes": "Because this property already has a boiler, we don't recommend HHR. We only have a " - "heating recommendation for an air source heat pump. Because the heating controls are " + "notes": "Because this property already has a boiler, we don't recommend HHR. We don't recommend an ashp " + "because the home is mid-terraced. Because the heating controls are " "Programmer, no room thermostat, we have a programmer, room thermostat and trvs recommendation" "for heating controls and for TTZC." }, @@ -306,7 +303,7 @@ testing_examples = [ 'uprn': 43088770.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention ' + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], @@ -408,10 +405,10 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Upgrade to a new condensing boiler upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], @@ -506,8 +503,8 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' @@ -557,8 +554,8 @@ testing_examples = [ }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', @@ -608,22 +605,13 @@ testing_examples = [ 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control)' ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' - ], + "heating_controls_recommendation_descriptions": [], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" }, @@ -667,8 +655,8 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' 'high heat retention storage controls however this is dependent on the existing system and may not be ' @@ -1258,8 +1246,8 @@ testing_examples = [ 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' @@ -1355,8 +1343,8 @@ testing_examples = [ 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' 'scheme grant', - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' @@ -1406,8 +1394,8 @@ testing_examples = [ 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], From 8ecd3e4bafebcbcc224c77383a9ed4aa3d95eb10 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 11:18:38 +0100 Subject: [PATCH 23/59] fixed the heating and heating controls unit tests --- .../test_data/heating_recommendations_data.py | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index e073ac99..8697e095 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -253,7 +253,7 @@ testing_examples = [ "heating_recommendation_descriptions": [ ], "heating_controls_recommendation_descriptions": [ - 'upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' ], @@ -354,7 +354,7 @@ testing_examples = [ 'scheme grant' ], "heating_controls_recommendation_descriptions": [ - 'upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' @@ -405,10 +405,10 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', - 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Upgrade to a new condensing boiler. upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], @@ -456,11 +456,14 @@ testing_examples = [ "heating_recommendation_descriptions": [ 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], "heating_controls_recommendation_descriptions": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" - "an air source heat pump" + "an air source heat pump and HHR (since if the home has a non-gas boiler, we recommend HHR)" }, { "epc": { @@ -503,16 +506,18 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + 'scheme grant', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" - "well as an air source heat pump" + "well as an air source heat pump and HHR (since the home has a room heater set up)" }, { "epc": { @@ -554,17 +559,15 @@ testing_examples = [ }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' ], "heating_controls_recommendation_descriptions": [], - "notes": "This property has assumed electric heaters. Boiler upgrade, HHR and ASHP are all recommended" + "notes": "This property has assumed electric heaters. Boiler upgrade, HHR are recommended. We don't recommend" + "an ASHP off of the bat because it's mid-terrace." }, { "epc": { @@ -1100,9 +1103,12 @@ testing_examples = [ 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" - "we recommend ASHP and HHR" + "we recommend ASHP and HHR. We also recommend upgrading the heating controls for the boiler" }, { "epc": { @@ -1152,7 +1158,10 @@ testing_examples = [ 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], "notes": "This property is a modified version of the previous dual heating property, where we lower the" "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" "storage heaters" From 781e19be992c17db3c57cc5da81b27491c846717 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 11:51:08 +0100 Subject: [PATCH 24/59] fixed some basic unit tests --- backend/Property.py | 3 +++ etl/sfr/midlands_portfolio_asset_list.py | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 etl/sfr/midlands_portfolio_asset_list.py diff --git a/backend/Property.py b/backend/Property.py index b15c9f25..0d194a79 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -187,6 +187,9 @@ class Property: # This additional condition data should change how we pass kwargs to this. We should no longer need to pass # kwargs to this class, but instead, we should pass the energy assessment condition data + energy_assessment = ( + {"condition": {}, "energy_assessment_is_newer": False} if energy_assessment is None else energy_assessment + ) self.energy_assessment_condition_data = energy_assessment["condition"] self.energy_assessment_is_newer = energy_assessment["energy_assessment_is_newer"] diff --git a/etl/sfr/midlands_portfolio_asset_list.py b/etl/sfr/midlands_portfolio_asset_list.py new file mode 100644 index 00000000..0434b45a --- /dev/null +++ b/etl/sfr/midlands_portfolio_asset_list.py @@ -0,0 +1,11 @@ +import pandas as pd + + +def app(): + """ + This script sets up + :return: + """ + + # Read in the portfolio EPC data + epc_data = pd.read_excel() From c693c6a633b09ce7a7464164f915c8f047cc6c9d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 12:19:28 +0100 Subject: [PATCH 25/59] updating write to db for solar api, wip --- backend/app/db/functions/solar_functions.py | 108 ++++++++++++-------- backend/app/plan/router.py | 1 - etl/sfr/midlands_portfolio_asset_list.py | 43 +++++++- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git a/backend/app/db/functions/solar_functions.py b/backend/app/db/functions/solar_functions.py index 59243f01..3ead1551 100644 --- a/backend/app/db/functions/solar_functions.py +++ b/backend/app/db/functions/solar_functions.py @@ -1,5 +1,6 @@ import datetime import pytz +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from backend.app.db.models.solar import Solar, SolarScenario @@ -38,57 +39,80 @@ def get_solar_data(session: Session, longitude: float = None, latitude: float = def store_batch_data(session: Session, api_data: dict, uprns_to_location: list, scenarios_data: list): """ This function will store the API data to the solar table against all of the UPRNs with longitude and latitude. + If a record already exists in the Solar table by UPRN, it will be updated instead of creating a new one. + Similarly, if a scenario exists in SolarScenario by number_panels, it will also be updated. + :param session: The database session :param api_data: The API data to store - :param uprns_to_location: A list of dictionaries containing uprn, longitude, and latitude + :param uprns_to_location: A list of dictionaries containing UPRN, longitude, and latitude :param scenarios_data: A list of dictionaries containing scenario data for each UPRN """ try: - - # Insert data into the Solar table and get the IDs - solar_records = [] + # Insert or update data into the Solar table for data in uprns_to_location: - solar_record = Solar( - uprn=data['uprn'], - longitude=data['longitude'], - latitude=data['latitude'], - google_api_response=api_data, - updated_at=datetime.datetime.now(pytz.utc) - ) - solar_records.append(solar_record) - session.add(solar_record) + existing_solar = session.execute(select(Solar).where(Solar.uprn == data['uprn'])).scalar_one_or_none() - session.flush() # Flush to get the IDs generated - - for record in solar_records: - session.refresh(record) # Refresh to populate the ID fields - - # Retrieve the IDs of the inserted records - inserted_ids = {record.uprn: record.id for record in solar_records} - - # Prepare the data for SolarScenario - scenario_records = [] - for data in uprns_to_location: - solar_id = inserted_ids.get(data['uprn']) - for scenario in scenarios_data: - scenario_record = SolarScenario( - solar_id=solar_id, - scenario_type=scenario['scenario_type'], - number_panels=scenario['number_panels'], - array_kwhp=scenario['array_kwhp'], - lifetime_dc_kwh=scenario['lifetime_dc_kwh'], - yearly_dc_kwh=scenario['yearly_dc_kwh'], - lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field - yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field - cost=scenario['cost'], - expected_payback_years=scenario.get('expected_payback_years'), # Optional field - panelled_roof_area=scenario['panelled_roof_area'], - is_default=scenario['is_default'] + if existing_solar: + # Update the existing record + existing_solar.longitude = data['longitude'] + existing_solar.latitude = data['latitude'] + existing_solar.google_api_response = api_data + existing_solar.updated_at = datetime.datetime.now(pytz.utc) + solar_id = existing_solar.id + else: + # Insert a new record + solar_record = Solar( + uprn=data['uprn'], + longitude=data['longitude'], + latitude=data['latitude'], + google_api_response=api_data, + updated_at=datetime.datetime.now(pytz.utc) ) - scenario_records.append(scenario_record) + session.add(solar_record) + session.flush() # Flush to get the IDs generated + session.refresh(solar_record) # Refresh to populate the ID field + solar_id = solar_record.id - # Insert data into the SolarScenario table - session.bulk_save_objects(scenario_records) + # Insert or update data in the SolarScenario table + for scenario in scenarios_data: + existing_scenario = session.execute( + select(SolarScenario).where( + SolarScenario.solar_id == solar_id, + SolarScenario.number_panels == scenario['number_panels'] + ) + ).scalar_one_or_none() + + if existing_scenario: + # Update the existing scenario record + existing_scenario.scenario_type = scenario['scenario_type'] + existing_scenario.array_kwhp = scenario['array_kwhp'] + existing_scenario.lifetime_dc_kwh = scenario['lifetime_dc_kwh'] + existing_scenario.yearly_dc_kwh = scenario['yearly_dc_kwh'] + existing_scenario.lifetime_ac_kwh = scenario.get('lifetime_ac_kwh') # Optional field + existing_scenario.yearly_ac_kwh = scenario.get('yearly_ac_kwh') # Optional field + existing_scenario.cost = scenario['cost'] + existing_scenario.expected_payback_years = scenario.get('expected_payback_years') # Optional field + existing_scenario.panelled_roof_area = scenario['panelled_roof_area'] + existing_scenario.is_default = scenario['is_default'] + else: + # Insert a new scenario record + scenario_record = SolarScenario( + solar_id=solar_id, + scenario_type=scenario['scenario_type'], + number_panels=scenario['number_panels'], + array_kwhp=scenario['array_kwhp'], + lifetime_dc_kwh=scenario['lifetime_dc_kwh'], + yearly_dc_kwh=scenario['yearly_dc_kwh'], + lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field + yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field + cost=scenario['cost'], + expected_payback_years=scenario.get('expected_payback_years'), # Optional field + panelled_roof_area=scenario['panelled_roof_area'], + is_default=scenario['is_default'] + ) + session.add(scenario_record) + + # Commit the changes after all operations session.commit() except Exception as e: diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e925fe00..b5cb96f6 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,7 +30,6 @@ from backend.app.plan.utils import get_cleaned from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc from backend.ml_models.api import ModelApi -from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.Property import Property from backend.apis.GoogleSolarApi import GoogleSolarApi diff --git a/etl/sfr/midlands_portfolio_asset_list.py b/etl/sfr/midlands_portfolio_asset_list.py index 0434b45a..01a01907 100644 --- a/etl/sfr/midlands_portfolio_asset_list.py +++ b/etl/sfr/midlands_portfolio_asset_list.py @@ -1,4 +1,5 @@ import pandas as pd +from utils.s3 import save_csv_to_s3 def app(): @@ -7,5 +8,45 @@ def app(): :return: """ + portfolio_id = 108 + # Read in the portfolio EPC data - epc_data = pd.read_excel() + epc_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" + ) + + asset_list = epc_data[ + [ + "ADDRESS1", "POSTCODE", "UPRN" + ] + ].copy().rename( + columns={ + "ADDRESS1": "address", + "POSTCODE": "postcode", + "UPRN": "uprn" + } + ) + + # Store data and prepare payload + + filename = f"{8}/{portfolio_id}/asset_list.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(portfolio_id), + "housing_type": "Private", + "goal": "Increasing EPC", + "goal_value": "C", + "trigger_file_path": filename, + "already_installed_file_path": "", + "patches_file_path": "", + "non_invasive_recommendations_file_path": "", + "budget": None, + "scenario_name": "EPC C Package", + "multi_plan": True, + } + print(body) From ef6aad6425d9b2b4c1c77e25eb5a037eb1da35eb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 12:23:33 +0100 Subject: [PATCH 26/59] overwrite solar and solar scenario storage to db instead of creating a new record --- backend/apis/GoogleSolarApi.py | 3 --- backend/app/plan/router.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 41ec7c11..a99d96c9 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -192,8 +192,6 @@ class GoogleSolarApi: if not self.need_to_store: return - logger.info("Storing to database") - scenarios_data = self.panel_performance.head(1)[ [ "n_panels", @@ -221,7 +219,6 @@ class GoogleSolarApi: scenarios_data["scenario_type"] = scenario_type scenarios_data = scenarios_data.to_dict(orient="records") - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it exists store_batch_data( session=session, api_data=self.insights_data, diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index b5cb96f6..d1578cc1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -651,8 +651,6 @@ async def trigger_plan(body: PlanTriggerRequest): ) # Store the data in the database - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it - # exists solar_api_client.save_to_db( session=session, uprns_to_location=[ From 7d907ce8c09117ae787d9e85d40b67426a02b836 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 13:32:42 +0100 Subject: [PATCH 27/59] fixing bug to allow all measures when no inclusions or exclusions are specified --- recommendations/Recommendations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index d2c1db1b..fbaf0f9b 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -79,7 +79,9 @@ class Recommendations: inclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.inclusions] exclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.exclusions] - if inclusions_full and exclusions_full: + # If inclusions and exclusions are empty, it means that nothing was specified, so we allow + # all recommendation types + if not inclusions_full and not exclusions_full: # All typical measures return self.all_specific_measures From 8a09a29956f2294d9d25855a378086ca682f9795 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 18:53:04 +0100 Subject: [PATCH 28/59] adding new installer costs for solar pv to cost class --- backend/app/plan/schemas.py | 15 ++++- recommendations/Costs.py | 77 +++++++++++++++++++---- recommendations/Recommendations.py | 27 ++++---- recommendations/SolarPvRecommendations.py | 3 +- 4 files changed, 96 insertions(+), 26 deletions(-) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 2968babf..68f8bbf5 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -20,7 +20,7 @@ SPECIFIC_MEASURES = [ # Walls "internal_wall_insulation", "external_wall_insulation", - "cavity_wall_insulation" + "cavity_wall_insulation", # Roof "loft_insulation", "flat_roof_insulation", @@ -32,7 +32,20 @@ SPECIFIC_MEASURES = [ "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump", + "secondary_heating", + # Solar + "solar_pv", + # Windows Glazing + "windows", + # Mechanical ventilation + "ventilation", + # Other + "low_energy_lighting", + "fireplace", + "hot_water", +] +NON_INVASIVE_SPECIFIC_MEASURES = [ # Specific measures that will typically come from an energy assessment "trickle_vents", "draught_proofing", diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 908a409a..71d20855 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -37,6 +37,30 @@ MCS_SOLAR_PV_COST_DATA = { "average_cost_per_kwh-Northern Ireland": 1347, } +INSTALLER_SOLAR_COSTS = [ + {'n_panels': 4, 'array_kwp': 1.6, 'cost': 3040.00, 'installer': 'CEG'}, + {'n_panels': 5, 'array_kwp': 2.1, 'cost': 3201.00, 'installer': 'CEG'}, + {'n_panels': 6, 'array_kwp': 2.5, 'cost': 3363.00, 'installer': 'CEG'}, + {'n_panels': 7, 'array_kwp': 2.9, 'cost': 3524.00, 'installer': 'CEG'}, + {'n_panels': 8, 'array_kwp': 3.3, 'cost': 3686.00, 'installer': 'CEG'}, + {'n_panels': 9, 'array_kwp': 3.7, 'cost': 3847.00, 'installer': 'CEG'}, + {'n_panels': 10, 'array_kwp': 4.1, 'cost': 4009.00, 'installer': 'CEG'}, + {'n_panels': 11, 'array_kwp': 4.5, 'cost': 4170.00, 'installer': 'CEG'}, + {'n_panels': 12, 'array_kwp': 4.9, 'cost': 4332.00, 'installer': 'CEG'}, + {'n_panels': 13, 'array_kwp': 5.3, 'cost': 4835.00, 'installer': 'CEG'}, + {'n_panels': 14, 'array_kwp': 5.7, 'cost': 5015.00, 'installer': 'CEG'}, + {'n_panels': 15, 'array_kwp': 6.2, 'cost': 5176.00, 'installer': 'CEG'}, + {'n_panels': 16, 'array_kwp': 6.6, 'cost': 5338.00, 'installer': 'CEG'}, + {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'}, + {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} +] + +INSTALLER_SCAFFOLDING_COSTS = [ + {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'}, + {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'}, + {'stories': 3, 'description': '3 Story Scaffold', 'cost': 1077.00, 'installer': 'CEG'} +] + # This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average, # to be conservative MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = { @@ -54,10 +78,27 @@ MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = { "Scotland": 12586, "Northern Ireland": 12000, # There are hardly any air source heat pump installs going on in Northern Ireland } + +INSTALLER_ASHP_COSTS = [ + {'capacity_kw': 5.0, 'brand': 'Mitsubishi', 'tank_size_liters': 150, 'cost': 10149.53, 'installer': 'CEG'}, + {'capacity_kw': 6.0, 'brand': 'Mitsubishi', 'tank_size_liters': 170, 'cost': 10823.48, 'installer': 'CEG'}, + {'capacity_kw': 8.5, 'brand': 'Mitsubishi', 'tank_size_liters': 200, 'cost': 11312.43, 'installer': 'CEG'}, + {'capacity_kw': 11.2, 'brand': 'Mitsubishi', 'tank_size_liters': 250, 'cost': 12156.75, 'installer': 'CEG'}, + {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'}, + {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'}, + {'capacity_kw': 17.0, 'brand': 'Grant', 'tank_size_liters': 300, 'cost': 14445.00, 'installer': 'CEG'}, + {'capacity_kw': 20.0, 'brand': 'Ecoforest', 'tank_size_liters': 400, 'cost': 21189.41, 'installer': 'CEG'}, + {'capacity_kw': None, 'brand': '2 x cascaded ASHPs', 'tank_size_liters': 500, 'cost': 22950.00, 'installer': 'CEG'} +] + BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500 -# This is based on quotes from installers -BATTERY_COST = 3500 +INSTALLER_SOLAR_BATTERY_COSTS = [ + {'capacity_kwh': 5, 'description': 'Battery Add on', 'cost': 2700.00, 'installer': 'CEG'}, + {'capacity_kwh': 10, 'description': 'Battery Add on', 'cost': 4300.00, 'installer': 'CEG'}, + {'capacity_kwh': 5, 'description': 'Battery Retrofit existing system', 'cost': 4250.00, 'installer': 'CEG'}, + {'capacity_kwh': 10, 'description': 'Battery Retrofit Existing system', 'cost': 5950.00, 'installer': 'CEG'} +] # This is based on https://www.checkatrade.com/blog/cost-guides/cost-smart-thermostat/ SMART_APPLIANCE_THERMOSTAT_COST = 400 @@ -1013,7 +1054,14 @@ class Costs: "labour_days": labour_days } - def solar_pv(self, wattage: float, has_battery: bool = False, array_cost=None): + def solar_pv( + self, wattage: float, + n_panels: int | float, + has_battery: bool = False, + array_cost=None, + n_floors: int = 1, + battery_kwh: int = 5, + ): """ Calculates the total cost for solar PV based data provided by the MCS dashboard, which contains @@ -1025,23 +1073,26 @@ class Costs: Price can also be benchmarked against this checkatrade article: https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/ - :param wattage: Peak wattage of the solar PV system] + :param wattage: Peak wattage of the solar PV system + :param n_panels: Number of solar panels :param has_battery: Bool, whether the system includes a battery :param array_cost: float, containing the cost of the solar PV array + :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding + :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5 """ - # Get the cost data relevant to the region - regional_cost = MCS_SOLAR_PV_COST_DATA["-".join(["average_cost_per_kwh", self.region])] + system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] - if array_cost is not None: - total_cost = array_cost - else: - kw = wattage / 1000 - total_cost = kw * regional_cost + total_cost = array_cost if array_cost is not None else system_cost if has_battery: - # The battery cost is based on the £3500 quote, recieved from installers - total_cost += BATTERY_COST + battery_cost = [c for c in INSTALLER_SOLAR_BATTERY_COSTS if c["capacity_kwh"] == battery_kwh][0]["cost"] + total_cost += battery_cost + + scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"] + total_cost += scaffolding_cost + + # We add an additional cost for scaffolding subtotal_before_vat = total_cost / (1 + self.VAT_RATE) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index fbaf0f9b..5037f450 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -18,9 +18,8 @@ from recommendations.DraughtProofingRecommendations import DraughtProofingRecomm from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.apis.GoogleSolarApi import GoogleSolarApi import backend.app.assumptions as assumptions -from backend.app.plan.schemas import TYPICAL_MEASURE_TYPES, SPECIFIC_MEASURES, MEASURE_MAP +from backend.app.plan.schemas import SPECIFIC_MEASURES, MEASURE_MAP, NON_INVASIVE_SPECIFIC_MEASURES -ASHP_COP = 3 STARTING_DUMMY_ID_VALUE = -9999 @@ -50,8 +49,11 @@ class Recommendations: self.exclusions = exclusions if exclusions else [] self.inclusions = inclusions if inclusions else [] - self.all_typical_measures = TYPICAL_MEASURE_TYPES self.all_specific_measures = SPECIFIC_MEASURES + self.all_non_invase_measures = NON_INVASIVE_SPECIFIC_MEASURES + self.non_invasive_recommendation_types = [ + r["type"] for r in self.property_instance.non_invasive_recommendations + ] self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials) self.wall_recomender = WallRecommendations(property_instance=property_instance, materials=materials) @@ -82,14 +84,18 @@ class Recommendations: # If inclusions and exclusions are empty, it means that nothing was specified, so we allow # all recommendation types if not inclusions_full and not exclusions_full: - # All typical measures - return self.all_specific_measures + # All typical measures - this does not include non-invasive measures inless they are specified + return self.all_specific_measures + self.non_invasive_recommendation_types if inclusions_full: return inclusions_full if exclusions_full: - return [m for m in self.all_specific_measures if m not in exclusions_full] + measures = [ + m for m in self.all_specific_measures + self.non_invasive_recommendation_types + if m not in exclusions_full + ] + return measures def recommend(self): @@ -146,11 +152,10 @@ class Recommendations: if self.draught_proofing_recommender.recommendation: property_recommendations.append(self.draught_proofing_recommender.recommendation) - if "floor_insulation" in measures: - self.floor_recommender.recommend(phase=phase, measures=measures) - if self.floor_recommender.recommendations: - property_recommendations.append(self.floor_recommender.recommendations) - phase += 1 + self.floor_recommender.recommend(phase=phase, measures=measures) + if self.floor_recommender.recommendations: + property_recommendations.append(self.floor_recommender.recommendations) + phase += 1 if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index d0d555c9..bbaffdda 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -196,7 +196,8 @@ class SolarPvRecommendations: cost_result = self.costs.solar_pv( wattage=recommendation_config["array_wattage"], has_battery=has_battery, - array_cost=non_invasive_recommendation.get("cost", None) + array_cost=non_invasive_recommendation.get("cost", None), + n_panels=recommendation_config["n_panels"], ) kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 if has_battery: From 503a19291dcacf7d1a4ab9b09107bd5a59c49b95 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 19:12:02 +0100 Subject: [PATCH 29/59] updating solar pv costs --- recommendations/Costs.py | 16 ++++++++++++++-- recommendations/SolarPvRecommendations.py | 11 ++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 71d20855..671c4db7 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -55,6 +55,11 @@ INSTALLER_SOLAR_COSTS = [ {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} ] +# CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone +# https://midsummerwholesale.co.uk/buy/solshare +INSTALLER_SOLAR_PV_INVERTER_COST = 7500 +INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST = 500 # Just a rough guess to labour costs + INSTALLER_SCAFFOLDING_COSTS = [ {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'}, {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'}, @@ -1055,12 +1060,13 @@ class Costs: } def solar_pv( - self, wattage: float, + self, n_panels: int | float, has_battery: bool = False, array_cost=None, n_floors: int = 1, battery_kwh: int = 5, + needs_inverter=False ): """ @@ -1073,12 +1079,13 @@ class Costs: Price can also be benchmarked against this checkatrade article: https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/ - :param wattage: Peak wattage of the solar PV system :param n_panels: Number of solar panels :param has_battery: Bool, whether the system includes a battery :param array_cost: float, containing the cost of the solar PV array :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5 + :param needs_inverter: Bool, whether the system needs an inverter, where the solar panels are feeding multiple + units """ system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] @@ -1092,6 +1099,11 @@ class Costs: scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"] total_cost += scaffolding_cost + if needs_inverter: + total_cost += INSTALLER_SOLAR_PV_INVERTER_COST + # We also add an additional labour cost + total_cost += INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST + # We add an additional cost for scaffolding subtotal_before_vat = total_cost / (1 + self.VAT_RATE) diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index bbaffdda..dc11ce4a 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -104,8 +104,13 @@ class SolarPvRecommendations: roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100) else: raise Exception("IMPLEMENT ME") - # Spread the cost to the individual units - adding a 20% contingency - total_cost = recommendation_config["total_cost"] / n_units + total_cost = self.costs.solar_pv( + array_cost=recommendation_config.get("cost", None), + n_panels=recommendation_config["n_panels"], + n_floors=self.property.number_of_storeys["number_of_storeys"], + needs_inverter=True, + )["total"] / n_units + kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 # Default to a weeks work for a team of 3 people doing 8 hour days labour_days = 5 @@ -194,10 +199,10 @@ class SolarPvRecommendations: roof_coverage_percent = np.ceil(roof_coverage_percent / 10) * 10 for has_battery in [False, True]: cost_result = self.costs.solar_pv( - wattage=recommendation_config["array_wattage"], has_battery=has_battery, array_cost=non_invasive_recommendation.get("cost", None), n_panels=recommendation_config["n_panels"], + n_floors=self.property.number_of_floors ) kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 if has_battery: From 767d0d3132a77549f06a64e7c7ee52bcf81923df Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 15:55:07 +0100 Subject: [PATCH 30/59] updating costing methodology for new installer costs --- backend/apis/GoogleSolarApi.py | 18 ++++-- recommendations/Costs.py | 77 +++++++------------------- recommendations/WallRecommendations.py | 15 ----- 3 files changed, 32 insertions(+), 78 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index a99d96c9..f1e3d2e9 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -42,6 +42,9 @@ class GoogleSolarApi: # your area installation_life_span = 20 + MIN_UNIT_PANELS = 4 # Minimum number of panels we allow for a domestic building + MIN_BUILDING_PANELS = 10 # Minimum number of panels we allow for a block of flats + def __init__(self, api_key, max_retries=5): """ Initialize the GoogleSolarApi class with the provided API key and maximum retries. @@ -250,6 +253,9 @@ class GoogleSolarApi: Optimise the solar panel configuration for the building. :return: """ + # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the + # minimum is 4 + min_panels = self.MIN_BUILDING_PANELS if is_building else self.MIN_UNIT_PANELS cost_instance = Costs(property_instance=property_instance) if property_instance is not None else None @@ -264,6 +270,10 @@ class GoogleSolarApi: roi_summary = [] for segment in roof_segment_summaries: + + if segment["panelsCount"] < min_panels: + continue + wattage = segment["panelsCount"] * self.insights_data["solarPotential"]["panelCapacityWatts"] generated_dc_energy = segment["yearlyEnergyDcKwh"] ratio = generated_dc_energy / wattage @@ -272,7 +282,9 @@ class GoogleSolarApi: cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000) else: cost = cost_instance.solar_pv( - wattage=wattage, has_battery=False + n_panels=segment["panelsCount"], + has_battery=False, + n_floors=property_instance.number_of_floors, )["total"] roi_summary.append( @@ -330,10 +342,6 @@ class GoogleSolarApi: # We can have duplicate configurations panel_performance = panel_performance.drop_duplicates() - # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the - # minimum is 4 - min_panels = 10 if is_building else 4 - panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels] if panel_performance.empty: self.panel_performance = pd.DataFrame( diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 671c4db7..c71316ad 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -54,6 +54,8 @@ INSTALLER_SOLAR_COSTS = [ {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'}, {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} ] +# This is the maximum number of panels that we have a cost from the installers for +INSTALLER_MAX_PANELS = 18 # CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone # https://midsummerwholesale.co.uk/buy/solshare @@ -362,7 +364,7 @@ class Costs: "labour_days": labour_days } - def internal_wall_insulation(self, wall_area, material, non_insulation_materials): + def internal_wall_insulation(self, wall_area, material): """ Broadly speaking, the high level steps to an internal wall insulation job are the following: @@ -401,74 +403,25 @@ class Costs: "labour_days": labour_days, } - # Extract and check the different types of data we'll need - demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] - vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] - redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] - if not demolition_data: - raise ValueError("No data found for iwi_wall_demolition") - - if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3): - raise ValueError("Incorrect number of data entries for non-insulation materials") - # Break out the individual material costs # Since we don't know the exact wall construction, we take an average for demolition costs, since # the cost will depend on the type of wall construction - demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) - insulation_material_costs = material["material_cost"] * wall_area - vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area - redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) - demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - - # Again for demolition, we average since we aren't sure which demolition process will be used - demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) - insulation_labour_costs = material["labour_cost"] * wall_area - vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area - redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) - - labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + - redecoration_labour_costs) - - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + - redecoration_material_costs) - - subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - - contingency_cost = subtotal_before_profit * self.IWI_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost - - demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) - insulation_labour_hours = material["labour_hours_per_unit"] * wall_area - vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area - redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) - - labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + - redecoration_labour_hours) + total_including_vat = material["total_cost"] * wall_area + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat + # We estimate 1 weeks worth of work + labour_hours = 160 # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people labour_days = (labour_hours / 8) / 4 return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, "labour_hours": labour_hours, "labour_days": labour_days, - "labour_cost": labour_costs } def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -1088,7 +1041,15 @@ class Costs: units """ - system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] + if n_panels > INSTALLER_MAX_PANELS: + base_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == INSTALLER_MAX_PANELS][0]["cost"] + cost_per_panel = [ + c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == (INSTALLER_MAX_PANELS - 1) + ][0]["cost"] + cost_per_panel = base_cost - cost_per_panel + system_cost = base_cost + (n_panels - INSTALLER_MAX_PANELS) * cost_per_panel + else: + system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] total_cost = array_cost if array_cost is not None else system_cost diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index a0c71860..0ddd7b0b 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -106,23 +106,10 @@ class WallRecommendations(Definitions): part for part in materials if part["type"] == "internal_wall_insulation" ] - self.internal_wall_non_insulation_materials = [ - part - for part in materials - if part["type"] - in ["iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"] - ] - self.external_wall_insulation_materials = [ part for part in materials if part["type"] == "external_wall_insulation" ] - self.external_wall_non_insulation_materials = [ - part - for part in materials - if part["type"] in ["ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"] - ] - def ewi_valid(self): """ This method check available data, to determine if a property is suitable for external wall insulation @@ -508,7 +495,6 @@ class WallRecommendations(Definitions): cost_result = self.costs.internal_wall_insulation( wall_area=self.property.insulation_wall_area, material=material.to_dict(), - non_insulation_materials=non_insulation_materials, ) already_installed = ( "internal_wall_insulation" @@ -617,7 +603,6 @@ class WallRecommendations(Definitions): iwi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials), - non_insulation_materials=self.internal_wall_non_insulation_materials, phase=phase, ) From fe86886adcef6225b31457fbaf5379745b30da82 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 15:57:06 +0100 Subject: [PATCH 31/59] refactoring costing function for solid wall insulation| --- recommendations/Costs.py | 166 +------------------------ recommendations/WallRecommendations.py | 20 ++- 2 files changed, 9 insertions(+), 177 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c71316ad..101de0dd 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -364,26 +364,9 @@ class Costs: "labour_days": labour_days } - def internal_wall_insulation(self, wall_area, material): + def solid_wall_insulation(self, wall_area, material): """ - Broadly speaking, the high level steps to an internal wall insulation job are the following: - - 1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles. - It's important to factor in the disposal of debris and the potential need for additional protective - measures to ensure the safety of the work area. - - 2) Insulation Installation: This is the core part of the process where the chosen insulation material is - applied. The choice of insulation material will depend on several factors including thermal performance, - wall construction, and space constraints. - - 3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation, - which can compromise its effectiveness and lead to mold growth. - - 4) Re-decoration: This involves applying plaster to the wall and then painting. - The quality of finish here is important for both aesthetic and functional reasons. - - 5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames, - or window sills might be necessary. + Implements costing methodology now that we have direct quotes from installers. :return: """ @@ -638,151 +621,6 @@ class Costs: "labour_cost": labour_costs } - def external_wall_insulation(self, wall_area, material, non_insulation_materials): - """ - We characterise external wall insulation as the following steps: - - 1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective - sheets to protect the flooring and landscaping around the work area. - - 2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or - detached houses. For terraced houses or lower-level work, scaffolding might not be necessary. - - 3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material, - and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement - of the top surface may be necessary. - - 4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation - boards and stabilize the wall surface, especially if it's old or weathered. - - 5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings, - or a combination of both. - - 6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation. - This layer provides strength and helps prevent cracking. - - 7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation - and provides an aesthetic look. - - 8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that - were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness. - - 9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system, - followed by cleaning up the site to remove all debris and materials. - - In the actual materials data, at this point, we have costing for: - - wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition) - - wall surface cleaning and priming (ewi_wall_preparation) - - insulation (external_wall_insulation) - - basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh) - - All of this data comes from SPONS, however there are some clear features missing. Because we could not find - suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding, - re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by - increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these - steps should be included in the materials data. We will look to improve this in the future, with data from - installers - - :param wall_area: - :param material: - :param non_insulation_materials: - :return: - """ - - if material["is_installer_quote"]: - total_cost = material["total_cost"] * wall_area - # Add on a buffer for scaffolding - if self.property.data["property-type"] == "House": - total_cost += self.EWI_SCAFFOLDING_PRELIMINARIES * total_cost - - labour_hours = material["labour_hours_per_unit"] * wall_area - - # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 - # people - labour_days = (labour_hours / 8) / 4 - - return { - "total": total_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - } - - # For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding - - if self.property.data["property-type"] == "House": - if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]: - preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES - else: - preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES - elif self.property.data["property-type"] in ["Maisonette", "Flat"]: - preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES - elif self.property.data["property-type"] == "Bungalow": - preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES - - demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] - preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] - redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] - - if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1): - raise ValueError("Incorrect number of data entries for non-insulation materials") - - # Break out the individual material costs - # Since we don't know the exact wall construction, we take an average for demolition costs, since - # the cost will depend on the type of wall construction - demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) - insulation_material_costs = material["material_cost"] * wall_area - preparation_material_costs = preparation_data[0]["material_cost"] * wall_area - redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area - - demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - - demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) - insulation_labour_costs = material["labour_cost"] * wall_area - preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area - redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area - - labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs + - preparation_labour_costs) - - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs + - redecoration_material_costs) - - subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - - contingency_cost = subtotal_before_profit * self.CONTINGENCY - preliminaries_cost = subtotal_before_profit * preliminaries_rate - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - vat_cost = subtotal_before_vat * self.VAT_RATE - total_cost = subtotal_before_vat + vat_cost - - demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) - insulation_labour_hours = material["labour_hours_per_unit"] * wall_area - preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area - redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area - - labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours + - preparation_labour_hours) - - # Assume a team of 3-5 people for a small to medium size project - labour_days = (labour_hours / 8) / 4 - - return { - "total": total_cost, - "subtotal": subtotal_before_vat, - "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - "labour_cost": labour_costs - } - def low_energy_lighting(self, number_of_lights, number_current_lel_lights, material): """ diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 0ddd7b0b..1c483bff 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -172,7 +172,6 @@ class WallRecommendations(Definitions): ewi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame(self.external_wall_insulation_materials), - non_insulation_materials=self.external_wall_non_insulation_materials, phase=phase ) @@ -437,7 +436,7 @@ class WallRecommendations(Definitions): return simulation_config - def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase): + def _find_insulation(self, u_value, insulation_materials, phase): lowest_selected_u_value = None recommendations = [] @@ -482,6 +481,11 @@ class WallRecommendations(Definitions): lowest_selected_u_value, new_u_value ) + cost_result = self.costs.solid_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + ) + if material["type"] == "internal_wall_insulation": if iwi_non_invasive_recommendations.get("cost") is not None: @@ -492,10 +496,6 @@ class WallRecommendations(Definitions): sap_points = iwi_non_invasive_recommendations.get("sap_points", None) survey = iwi_non_invasive_recommendations.get("survey", False) - cost_result = self.costs.internal_wall_insulation( - wall_area=self.property.insulation_wall_area, - material=material.to_dict(), - ) already_installed = ( "internal_wall_insulation" in self.property.already_installed @@ -511,12 +511,7 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - - cost_result = self.costs.external_wall_insulation( - wall_area=self.property.insulation_wall_area, - material=material.to_dict(), - non_insulation_materials=non_insulation_materials, - ) + already_installed = ( "external_wall_insulation" in self.property.already_installed @@ -594,7 +589,6 @@ class WallRecommendations(Definitions): insulation_materials=pd.DataFrame( self.external_wall_insulation_materials ), - non_insulation_materials=self.external_wall_non_insulation_materials, phase=phase, ) From d0f7c7f63a850cd830ca5716f6b7054e261a744f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 16:08:06 +0100 Subject: [PATCH 32/59] updating cwi costs --- recommendations/Costs.py | 37 +++++++------------------- recommendations/WallRecommendations.py | 2 +- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 101de0dd..e1b16899 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -257,7 +257,6 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # CWI usually takes 1 day labour_hours = 8 labour_days = 1 @@ -272,40 +271,22 @@ class Costs: "labour_days": labour_days, } - material_cost_per_m2 = material["material_cost"] - - base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor - - subtotal_before_profit = base_material_cost + labour_cost - - contingency_cost = subtotal_before_profit * self.CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost + total_including_vat = material["total_cost"] * wall_area if is_extraction_and_refill: - # bump up the cost of the work - total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area + total_including_vat = CAVITY_EXTRACTION_COST * wall_area # Additional 2 days work - labour_hours = labour_hours + (2 * 8) - labour_days = labour_days + 2 + labour_hours += + (2 * 8) + labour_days += + 2 + + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": base_material_cost, - "profit": profit_cost, "labour_hours": labour_hours, - "labour_cost": labour_cost, "labour_days": labour_days } diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 1c483bff..69bfdfb4 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -511,7 +511,7 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - + already_installed = ( "external_wall_insulation" in self.property.already_installed From 0c6d8121c2ccaf0075c5570c6e721af15b7fbf0b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 10:58:59 +0100 Subject: [PATCH 33/59] updating costing for loft insulation --- recommendations/Costs.py | 44 ++++++-------------------- recommendations/RoofRecommendations.py | 4 +-- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index e1b16899..69ff6073 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -298,51 +298,25 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - labour_hours = material["labour_hours_per_unit"] * floor_area - # Assume a team of 1 person - labour_days = labour_hours / 8 - if material["is_installer_quote"]: total_cost = material["total_cost"] * floor_area return { "total": total_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, + "labour_hours": 8, + "labour_days": 1, } - material_cost_per_m2 = material["material_cost"] - - # We inflate material costs due to recent price increases - material_cost_per_m2 = material_cost_per_m2 * 1.5 - - base_material_cost = material_cost_per_m2 * floor_area - labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor - - subtotal_before_profit = base_material_cost + labour_cost - - # We use high risk contingency because of the possibility of access issues and clearing existing insulation - contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost + total_including_vat = material["total_cost"] * floor_area + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": base_material_cost, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_cost": labour_cost, - "labour_days": labour_days + "labour_hours": 8, + "labour_days": 1 } def solid_wall_insulation(self, wall_area, material): diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 6635dd51..8c7e2291 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -44,10 +44,11 @@ class RoofRecommendations: self.recommendations = [] self.loft_insulation_materials = [ - part for part in materials if part["type"] == "loft_insulation" + part for part in materials if (part["type"] == "loft_insulation") and (part["is_installer_quote"]) ] self.loft_non_insulation_materials = [] + # We don't have proper installer quotes for flat roof insulation self.flat_roof_insulation_materials = [ part for part in materials if part["type"] == "flat_roof_insulation" ] @@ -266,7 +267,6 @@ class RoofRecommendations: lowest_selected_u_value = None recommendations = [] for _, insulation_material_group in insulation_materials.groupby("description"): - for _, material in insulation_material_group.iterrows(): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the From e6fc34741c9b58cb2de1d5091b3200c4f17ff3cb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 14:35:35 +0100 Subject: [PATCH 34/59] update flat roof insulation recommendations --- recommendations/Costs.py | 90 +-------------------- recommendations/FireplaceRecommendations.py | 2 +- recommendations/RoofRecommendations.py | 33 +++----- 3 files changed, 12 insertions(+), 113 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 69ff6073..08b05a8a 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -290,9 +290,9 @@ class Costs: "labour_days": labour_days } - def loft_insulation(self, floor_area, material): + def loft_and_flat_insulation(self, floor_area, material): """ - Calculates the total cost for cavity wall insulation based on material and labor costs, + Calculates the total cost for loft/flat roof insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. :return: A dictionary containing detailed cost breakdown. @@ -624,92 +624,6 @@ class Costs: "labour_cost": labour_cost } - def flat_roof_insulation(self, floor_area, material, non_insulation_materials): - """ - A model of a warm, flat roof construction can be seen in this video: - https://www.youtube.com/watch?v=WZ6Ng6YI9OA - Warm, flat roof insulation will normally be 100-125mm in depth - - We break this measure down into the following jobs to be done - 1) Preparation of the room. This involves cleaning the existing roof surface, removing any debris and repairing - any damage. Additionally, an edge barrier will likely need to be installed, to protect the sides of the - roof from water ingress. - 2) Primer Application. A layer of primer is applied to the clean roof surface to enhance the adhestia of - subsequent layers, and seal the existing roof surface. - 3) Vapour Proof Layer Installation. Lay a vapour control layer to prevent moisture ingress from inside the - building, which is essential in warm roof construction. - 4) Insulation Layer Application. Place and securely fix insulation boards over the roof. These could be rigid - boards like PIR (Polyisocyanurate). - 5) Waterproofing Membrane Installation: Cover the insulation (and timber layer, if used) with a - waterproofing membrane, like EPDM, PVC, or bituminous felt. Carefully seal all joints, edges, and around any - roof penetrations to ensure water tightness - - :param floor_area: Area of the flat roof to be insulated, based on the area of the floor - :param material: Selected insulation material - :param non_insulation_materials: Non-insulation materials required for the job - :return: - """ - - preparation_data_m2 = [ - x for x in non_insulation_materials if - (x["type"] == "flat_roof_preparation") and (x["cost_unit"] == "gbp_per_m2") - ] - vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_vapour_barrier"] - waterproofing_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_waterproofing"] - - if (len(preparation_data_m2) != 2) or (len(vapour_barrier_data) != 1) or ( - len(waterproofing_data) != 1): - raise ValueError("Incorrect number of data entries for non-insulation materials") - - # Break out the individual material costs - preparation_m2_material_costs = sum([x["material_cost"] * floor_area for x in preparation_data_m2]) - vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * floor_area - insulation_material_costs = material["material_cost"] * floor_area - - preparation_m2_labour_costs = sum([x["labour_cost"] * floor_area for x in preparation_data_m2]) - vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * floor_area - - # For waterproofing and upstand, we only have a total cost - waterproofing_total_costs = waterproofing_data[0]["total_cost"] * floor_area - - labour_costs = preparation_m2_labour_costs + vapour_barrier_labour_costs - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = preparation_m2_material_costs + vapour_barrier_material_costs + insulation_material_costs - - subtotal_before_profit = labour_costs + materials_costs + waterproofing_total_costs - - contingency_cost = subtotal_before_profit * self.FLAT_ROOF_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - vat_cost = subtotal_before_vat * self.VAT_RATE - total_cost = subtotal_before_vat + vat_cost - - preparation_m2_labour_hours = sum([x["labour_hours_per_unit"] * floor_area for x in preparation_data_m2]) - vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * floor_area - waterproofing_labour_hours = waterproofing_data[0]["labour_hours_per_unit"] * floor_area - - labour_hours = preparation_m2_labour_hours + vapour_barrier_labour_hours + waterproofing_labour_hours - - # To install flat roof insulation, assume a small/medium project might be conducted by a team of 2-4. - # We'll assume a team of 2 since a lot of the roofs will be on the smaller side and will review this later - labour_days = (labour_hours / 8) / 2 - - return { - "total": total_cost, - "subtotal": subtotal_before_vat, - "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - "labour_cost": labour_costs - } - def window_glazing(self, number_of_windows, material, is_secondary_glazing=False): """ We characterise the jobs to be done for window glazing as the following: diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 9a9d7f76..163728dd 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -9,7 +9,7 @@ class FireplaceRecommendations(Definitions): """ # This is our base assumption for the cost of the work - COST_OF_WORK = 300 + COST_OF_WORK = 235 def __init__( self, diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 8c7e2291..89b8205f 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -46,19 +46,12 @@ class RoofRecommendations: self.loft_insulation_materials = [ part for part in materials if (part["type"] == "loft_insulation") and (part["is_installer_quote"]) ] - self.loft_non_insulation_materials = [] # We don't have proper installer quotes for flat roof insulation self.flat_roof_insulation_materials = [ part for part in materials if part["type"] == "flat_roof_insulation" ] - self.flat_roof_non_insulation_materials = [ - part for part in materials if part["type"] in [ - "flat_roof_preparation", "flat_roof_vapour_barrier", "flat_roof_waterproofing" - ] - ] - # Extract the insulation thickness from the roof, which is used throughout this method self.insulation_thickness = convert_thickness_to_numeric( self.property.roof["insulation_thickness"], @@ -252,10 +245,8 @@ class RoofRecommendations: if is_pitched: insulation_materials = self.loft_insulation_materials - non_insulation_materials = self.loft_non_insulation_materials elif is_flat: insulation_materials = self.flat_roof_insulation_materials - non_insulation_materials = self.flat_roof_non_insulation_materials else: raise ValueError("Roof is not pitched or flat") @@ -297,14 +288,16 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + cost_result = self.costs.loft_and_flat_insulation( + floor_area=self.property.insulation_floor_area, + material=material + ) + + already_installed = material["type"] in self.property.already_installed + if already_installed: + cost_result = override_costs(cost_result) + if material["type"] == "loft_insulation": - cost_result = self.costs.loft_insulation( - floor_area=self.property.insulation_floor_area, - material=material - ) - already_installed = "loft_insulation" in self.property.already_installed - if already_installed: - cost_result = override_costs(cost_result) new_thickness = insulation_thickness + material["depth"] # This is based on the values we have in the training data @@ -341,14 +334,6 @@ class RoofRecommendations: new_description = f"Pitched, {int(proposed_depth)}mm loft insulation" elif material["type"] == "flat_roof_insulation": - cost_result = self.costs.flat_roof_insulation( - floor_area=self.property.insulation_floor_area, - material=material, - non_insulation_materials=non_insulation_materials - ) - already_installed = "flat_roof_insulation" in self.property.already_installed - if already_installed: - cost_result = override_costs(cost_result) new_description = "Flat, insulated" new_efficiency = "Good" else: From 4a216fb42301b781b8f95ba6aee55bb816ed1759 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 15:43:33 +0100 Subject: [PATCH 35/59] handling property with u-values in survey which is not a new dwelling --- etl/costs/app.py | 2 ++ recommendations/HeatingRecommender.py | 17 ++++++++--------- recommendations/RoofRecommendations.py | 2 +- recommendations/VentilationRecommendations.py | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/etl/costs/app.py b/etl/costs/app.py index 85c2410e..797191d2 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -82,6 +82,7 @@ def app(): db_engine = create_engine(db_string, pool_size=5, max_overflow=5) cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0) + ventilation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="Ventilation", header=0) loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0) iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0) suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0) @@ -95,6 +96,7 @@ def app(): costs = pd.concat( [ cwi_costs, + ventilation_costs, loft_insulation_costs, iwi_costs, suspended_floor_costs, diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 3daf0268..b54f89bb 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1034,17 +1034,16 @@ class HeatingRecommender: # Overwrite the existing boiler recommendation self.heating_recommendations.extend(combined_recommendations) else: - # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade - # but we'll only upgrade if we have a heating recommendation - has_heating_recommendation = any( - rec["type"] == "heating" for rec in self.heating_recommendations - ) - if has_heating_recommendation: - recommendation_phase += 1 - # The heating controls recommendation is distrinct from the boiler upgrade recommendation - # We insert phase into the recommendations for heating controls + # We consider a heating control upgrade as a measure which occures in the same phase as a boiler upgrade + # Namely, we have the following options within this phase + # 1) Boiler + heating controls + # 2) Boiler only + # 3) Heating controls only + # But they are options that are not mutually exclusive + # So, we actually set heating controls as a heating recommendation for recommendation in controls_recommender.recommendation: recommendation["phase"] = recommendation_phase + # recommendation["type"] = "heating" self.heating_control_recommendations.extend(controls_recommender.recommendation) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 89b8205f..fbd99d67 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -132,7 +132,7 @@ class RoofRecommendations: # The Roof is already compliant return - if self.property.data["transaction-type"] == "new dwelling": + if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]: return raise NotImplementedError("Implement me") diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 34439827..5913ab9c 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -52,7 +52,7 @@ class VentilationRecommendations(Definitions): already_installed = "cavity_wall_insulation" in self.property.already_installed - estimated_cost = n_units * part[0]["cost"] if not already_installed else 0 + estimated_cost = n_units * part[0]["total_cost"] if not already_installed else 0 labour_hours = 4 * n_units if not already_installed else 0 labour_days = 4 * n_units / 8.0 if not already_installed else 0 From 7236ff2ed2dfd86fe2ab7b7ade4d6636003dd9c1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 16:47:21 +0100 Subject: [PATCH 36/59] ensure panel performance meets minimum number of panels in the case of a double roof segment --- backend/apis/GoogleSolarApi.py | 1 + recommendations/WallRecommendations.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index f1e3d2e9..c82c9c9a 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -491,6 +491,7 @@ class GoogleSolarApi: panel_performance["n_panels"] = panel_performance["n_panels_halved"] panel_performance = panel_performance.drop(columns=["n_panels_halved"]) + panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels] self.panel_performance = panel_performance diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 69bfdfb4..4902ae03 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -69,6 +69,7 @@ class WallRecommendations(Definitions): "Timber frame, as built, no insulation": "Timber frame, with external insulation", 'Timber frame, as built, partial insulation': 'Timber frame, with external insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with external insulation", + "Sandstone, as built, no insulation": "Sandstone, with external insulation", } # These are the ending descriptions we consider for walls with internal insulation @@ -83,6 +84,7 @@ class WallRecommendations(Definitions): "Timber frame, as built, no insulation": "Timber frame, with internal insulation", 'Timber frame, as built, partial insulation': 'Timber frame, with internal insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with internal insulation", + "Sandstone, as built, no insulation": "Sandstone, with internal insulation", } def __init__( @@ -486,6 +488,10 @@ class WallRecommendations(Definitions): material=material.to_dict(), ) + already_installed = material["type"] in self.property.already_installed + if already_installed: + cost_result = override_costs(cost_result) + if material["type"] == "internal_wall_insulation": if iwi_non_invasive_recommendations.get("cost") is not None: @@ -496,13 +502,6 @@ class WallRecommendations(Definitions): sap_points = iwi_non_invasive_recommendations.get("sap_points", None) survey = iwi_non_invasive_recommendations.get("survey", False) - already_installed = ( - "internal_wall_insulation" - in self.property.already_installed - ) - if already_installed: - cost_result = override_costs(cost_result) - new_description = self.get_internal_external_wall_description( self.INTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value ) @@ -512,13 +511,6 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - already_installed = ( - "external_wall_insulation" - in self.property.already_installed - ) - if already_installed: - cost_result = override_costs(cost_result) - new_description = self.get_internal_external_wall_description( self.EXTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value ) From cf5f69d6f09b53d7a505cd436c7daab2adf5e517 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:31:50 +0100 Subject: [PATCH 37/59] debugging cleaning class for examples that hadn't been covered previously --- etl/epc_clean/app.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 6 ++++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/app.py b/etl/epc_clean/app.py index 1d833b72..1dfdd452 100644 --- a/etl/epc_clean/app.py +++ b/etl/epc_clean/app.py @@ -44,7 +44,8 @@ def app(): # Rename the columns to the same format as the api returns data.columns = [c.replace("_", "-").lower() for c in data.columns] # Take just date before the date threshold - data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] + # For this cleaning dataset, let's try and use all EPCs + # data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] # Convert to list of dictioaries as returned by the api data = data.to_dict("records") diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 56115dca..a7b4305e 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -66,6 +66,7 @@ class MainHeatAttributes(Definitions): "electric heat pumps": "electric heat pump", "solar-assisted heat pump": "solar assisted heat pump", "portable electric heating": "portable electric heaters", + "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", } edge_case_result = {} @@ -138,6 +139,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == ', electric': + self.edge_case_result['has_electric'] = True + self.is_edge_case = True + return + def process(self) -> Dict[str, Union[str, bool]]: result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 46fff6d8..b3cc4df4 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -75,6 +75,7 @@ class MainheatControlAttributes(Definitions): TO_REMAP = { "celect control": 'celect-type control', "celect controls": 'celect-type control', + "trv's, program & flow switch": 'trvs, programmer & flow switch', } WELSH_TEXT = { From 3957f0fcf8344367dc2fcd2c71e50900a1ef37aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:41:18 +0100 Subject: [PATCH 38/59] fixing cleaning descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 2 files changed, 6 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index a7b4305e..16897133 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -144,6 +144,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == 'community, community': + self.edge_case_result['has_community_scheme'] = True + self.is_edge_case = True + return + def process(self) -> Dict[str, Union[str, bool]]: result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index b3cc4df4..2759096d 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -76,6 +76,7 @@ class MainheatControlAttributes(Definitions): "celect control": 'celect-type control', "celect controls": 'celect-type control', "trv's, program & flow switch": 'trvs, programmer & flow switch', + 'appliance thermostat': 'appliance thermostats', } WELSH_TEXT = { From e942d6e70094d862255911c95af245c4673abdd8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:57:10 +0100 Subject: [PATCH 39/59] adding some additional case coverage to epc cleaner --- etl/epc_clean/app.py | 3 +-- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 7 +++---- .../tests/test_data/test_roof_attributes_cases.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/etl/epc_clean/app.py b/etl/epc_clean/app.py index 1dfdd452..a3c1018f 100644 --- a/etl/epc_clean/app.py +++ b/etl/epc_clean/app.py @@ -44,8 +44,7 @@ def app(): # Rename the columns to the same format as the api returns data.columns = [c.replace("_", "-").lower() for c in data.columns] # Take just date before the date threshold - # For this cleaning dataset, let's try and use all EPCs - # data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] + data = data[data["lodgement-date"] >= "2011-01-01"] # Convert to list of dictioaries as returned by the api data = data.to_dict("records") diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index f9cec48b..67f5bebd 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -96,6 +96,7 @@ class HotWaterAttributes(Definitions): WELSH_TEXT = { "ogçör brif system": "from main system", + "o r brif system": "from main system", "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery", "bwyler/cylchredydd nwy": "gas boiler/circulator", "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 84d1f3e9..154fe41b 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -6,7 +6,7 @@ from etl.epc_clean.epc_attributes.attribute_utils import extract_component_types class RoofAttributes(Definitions): ROOF_TYPES = ['pitched', 'roof room', 'loft', 'flat', 'thatched', 'at rafters', 'assumed'] - DWELLING_ABOVE = ["another dwelling above", "other premises above"] + DWELLING_ABOVE = ["another dwelling above", "other premises above", "other dwelling above"] WELSH_TEXT = { "ar oleddf, dim inswleiddio": "pitched, no insulation", @@ -113,9 +113,8 @@ class RoofAttributes(Definitions): # roof type result, description = extract_component_types(result, description, list_of_components=self.ROOF_TYPES) - result["has_dwelling_above"] = ( - "another dwelling above" in description or "other premises above" in description - ) + result["has_dwelling_above"] = any([x in description for x in self.DWELLING_ABOVE]) + for dwelling_above in self.DWELLING_ABOVE: description = description.replace(dwelling_above, "") diff --git a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py index 6b719afd..06c1f078 100644 --- a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py @@ -397,7 +397,7 @@ clean_roof_test_cases = [ 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'}, {'original_description': 'Average thermal transmittance 0.80 W/m+é-¦K', 'thermal_transmittance': 0.8, - 'thermal_transmittance_unit': 'w/m+é-¦k', 'is_pitched': False, 'is_roof_room': False, + 'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': None} ] From f3f04de3444ec0cdd1367b498e08b3330dbe7ce4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:01:35 +0100 Subject: [PATCH 40/59] adding welsh translation --- etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 154fe41b..a67a6029 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -16,6 +16,7 @@ class RoofAttributes(Definitions): "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', + "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", From 98272b9de45b59a7a5ff6cd2561c07d57d8c3a77 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:06:42 +0100 Subject: [PATCH 41/59] Adding floor edge case --- etl/epc_clean/epc_attributes/FloorAttributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 817c2b43..c9c2abab 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -11,7 +11,7 @@ class FloorAttributes(Definitions): # For the short term, while we are still exploring the data, we maintain a list of error cases which # we want to ignore and consider as no data. - OBSERVED_ERRORS = ["Conservatory"] + OBSERVED_ERRORS = ["Conservatory", "insulated"] WELSH_TEXT = { "(anheddiad arall islaw)": "(another dwelling below)", From 18dc0c109fb50a55baeae83bc5e592e22b3ceadc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:20:40 +0100 Subject: [PATCH 42/59] Added welsh translations --- etl/epc_clean/epc_attributes/LightingAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/etl/epc_clean/epc_attributes/LightingAttributes.py b/etl/epc_clean/epc_attributes/LightingAttributes.py index 18475b2d..52baa033 100644 --- a/etl/epc_clean/epc_attributes/LightingAttributes.py +++ b/etl/epc_clean/epc_attributes/LightingAttributes.py @@ -7,6 +7,7 @@ from etl.epc_clean.utils import correct_spelling class LightingAttributes(Definitions): WELSH_TEXT = { "goleuadau ynni-isel ym mhob un ogçör mannau gosod": "low energy lighting in all fixed outlets", + "goleuadau ynni-isel ym mhob un o r mannau gosod": "low energy lighting in all fixed outlets", "dim goleuadau ynni-isel": "no low energy lighting", "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets' } diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index a67a6029..453ada18 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -12,6 +12,7 @@ class RoofAttributes(Definitions): "ar oleddf, dim inswleiddio": "pitched, no insulation", "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)", "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", + "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", @@ -26,6 +27,7 @@ class RoofAttributes(Definitions): "(eiddo arall uwchben)": "(another dwelling above)", "(annedd arall uwchben)": "(another dwelling above)", "ystafell(oedd) to, wedigçöi hinswleiddio": "roof room(s), insulated", + "ystafell(oedd) to, wedi?i hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)", "ystafell(oedd) to, wedigçöi hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)", "ystafell(oedd) to, inswleiddio cyfyngedig (rhagdybiaeth)": "roof room(s), limited insulation (assumed)", "ystafell(oedd) to, inswleiddio cyfyngedig": "roof room(s), limited insulation", From 9fe9586d06170195d2aadd95c3ac3963d602716b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:40:02 +0100 Subject: [PATCH 43/59] adding new mainfuel community heating description --- etl/epc_clean/epc_attributes/MainFuelAttributes.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/MainFuelAttributes.py b/etl/epc_clean/epc_attributes/MainFuelAttributes.py index 72b86482..9bb53ff1 100644 --- a/etl/epc_clean/epc_attributes/MainFuelAttributes.py +++ b/etl/epc_clean/epc_attributes/MainFuelAttributes.py @@ -50,7 +50,8 @@ class MainFuelAttributes(Definitions): NO_INDIVIDUAL_HEATING_OR_COMMUNITY_NETWORK = [ 'to be used only when there is no heatinghotwater system or data is from a community network', - 'to be used only when there is no heatinghotwater system' + 'to be used only when there is no heatinghotwater system', + 'community heating schemes waste heat from power stations', ] def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 16897133..2d482125 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -67,6 +67,7 @@ class MainHeatAttributes(Definitions): "solar-assisted heat pump": "solar assisted heat pump", "portable electric heating": "portable electric heaters", "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", + "electric storage, electric": "electric storage heaters", } edge_case_result = {} @@ -98,6 +99,10 @@ class MainHeatAttributes(Definitions): self.description = remapped + backup_remap = self.REMAP.get(self.description) + if backup_remap: + self.description = backup_remap + self.process_edge_cases() if not self.nodata: From 78f5532dd9236507accf96416afa9e99dc0552ad Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:59:28 +0100 Subject: [PATCH 44/59] additional hot water system handled --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + .../tests/test_data/test_hot_water_attributes_cases.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index 67f5bebd..bd98ec50 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -19,6 +19,7 @@ class HotWaterAttributes(Definitions): 'solid fuel boiler', # burns solid materials to generate heat for water heating and/or space heating 'solid fuel range cooker', 'room heaters', # Generic/unspecified category + 'electric multipoint', ] # SYSTEM_TYPES refer to the larger system within which the heater operates. diff --git a/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py b/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py index 2d6f10a0..ae5348be 100644 --- a/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py @@ -219,4 +219,9 @@ hotwater_cases = [ 'heater_type': 'electric instantaneous', 'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': 'waste water heat recovery', 'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None}, + {'original_description': 'Electric multipoint', 'heater_type': 'electric multipoint', 'system_type': None, + 'thermostat_characteristics': None, + 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None, + 'distribution_system': None, 'no_system_present': None, 'appliance': None, 'assumed': False} + ] From 504d46c929cf7050c4660682498429e7a2c0a533 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 20:29:56 +0100 Subject: [PATCH 45/59] handling additional error cases --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + .../test_data/test_mainheat_attributes_cases.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index bd98ec50..78ee5f7d 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -101,6 +101,7 @@ class HotWaterAttributes(Definitions): "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery", "bwyler/cylchredydd nwy": "gas boiler/circulator", "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", + "o r brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", "twymwr tanddwr, an-frig": "electric immersion, off-peak", "ogçör brif system, gydag ynnigçör haul": "from main system, plus solar", "twymwr tanddwr, tarriff safonol": "electric immersion, standard tariff", diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 2d482125..bf86f573 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -68,6 +68,7 @@ class MainHeatAttributes(Definitions): "portable electric heating": "portable electric heaters", "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", "electric storage, electric": "electric storage heaters", + "radiator heating, electric": "room heaters, electric", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 453ada18..f3a4ee49 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -12,6 +12,7 @@ class RoofAttributes(Definitions): "ar oleddf, dim inswleiddio": "pitched, no insulation", "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)", "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", + "ar oleddf, wedi?i inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 558b176e..86175d4e 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1664,5 +1664,19 @@ mainheat_cases = [ 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': True, + 'has_underfloor_heating': False}, + {'original_description': 'Radiator heating, electric', 'has_radiators': False, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False} + ] From 3ebda6277e1c31feef898ffb20482948f5979bf7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 20:53:24 +0100 Subject: [PATCH 46/59] fixing multiple translations and issues with epc descriptions --- .../epc_attributes/MainheatAttributes.py | 2 ++ .../epc_attributes/RoofAttributes.py | 27 ++++++++++--------- .../test_mainheat_attributes_cases.py | 17 +++++++++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index bf86f573..c9f9fbe3 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -69,6 +69,8 @@ class MainHeatAttributes(Definitions): "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", "electric storage, electric": "electric storage heaters", "radiator heating, electric": "room heaters, electric", + "hot-water-only systems, gas": "no system present, electric heaters assumed", + "gas-fired heat pumps, electric": "air source heat pump, electric", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index f3a4ee49..f36d445f 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -66,10 +66,18 @@ class RoofAttributes(Definitions): search for regular expressions and translate """ - loft_insulation_thickness_match = re.search(r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", self.description) - loft_insulation_thickness_match2 = re.search(r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", self.description) - loft_insulation_thickness_match3 = re.search(r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", - self.description) + loft_insulation_regexes = [ + r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", + r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", + r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", + r"ar oleddf, (\d+mm) o inswleiddio yn y llofft", + r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft" + ] + li_thickness_match = None + for regex in loft_insulation_regexes: + li_thickness_match = re.search(regex, self.description) + if li_thickness_match: + break uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description) uvalue_search2 = re.search( @@ -77,15 +85,8 @@ class RoofAttributes(Definitions): ) # Step 2: Generalized translation with placeholder - if (loft_insulation_thickness_match is not None) | \ - (loft_insulation_thickness_match2 is not None) | \ - (loft_insulation_thickness_match3 is not None): - if loft_insulation_thickness_match is not None: - insulation_thickness = loft_insulation_thickness_match.group(1) - elif loft_insulation_thickness_match2 is not None: - insulation_thickness = loft_insulation_thickness_match2.group(1) - else: - insulation_thickness = loft_insulation_thickness_match3.group(1) + if li_thickness_match is not None: + insulation_thickness = li_thickness_match.group(1) self.description = f"pitched, {insulation_thickness} loft insulation" elif uvalue_search is not None or uvalue_search2 is not None: diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 86175d4e..82675a74 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1677,6 +1677,21 @@ mainheat_cases = [ 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, - 'has_underfloor_heating': False} + 'has_underfloor_heating': False}, + { + 'original_description': 'Hot-Water-Only Systems, gas', + 'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': True, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, + 'has_underfloor_heating': False + } ] From f0e7aa6d6b12855fdf0e3a2f09872445f365ba9c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 09:07:50 +0100 Subject: [PATCH 47/59] added additional welsh translation --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index c9f9fbe3..6382238f 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -34,6 +34,8 @@ class MainHeatAttributes(Definitions): "gwresogyddion ystafell, trydan": "room heaters, electric", "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan": "air source heat pump, underfloor heating, " "electric", + "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan, pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, " + "trydan": "air source heat pump, underfloor heating, electric", "cynllun cymunedol": "community scheme", "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas", "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs', From e0f0042086b13307f441503b707927fe4def6f9d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 09:23:42 +0100 Subject: [PATCH 48/59] welsh translations --- etl/epc_clean/epc_attributes/FloorAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatAttributes.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index c9c2abab..9d0f514d 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -30,6 +30,7 @@ class FloorAttributes(Definitions): "i ofod heb ei wresogi, wedigçöi inswleiddio": "to unheated space, insulated", "solet, wedigçöi inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "solet, wedigçöi inswleiddio": "solid, insulated", + "solet, wedi???i inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation" } diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 6382238f..430b418d 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -61,6 +61,8 @@ class MainHeatAttributes(Definitions): "bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage " "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", + # Should be handled by edge cases + ", trydan": ", electric", } REMAP = { From a27d664a2f3ed141473c759c54df112142ad07cc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 11:24:55 +0100 Subject: [PATCH 49/59] Handling windows cleaning edge case --- etl/epc_clean/epc_attributes/WindowAttributes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index e9139510..a52977e6 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -33,12 +33,18 @@ class WindowAttributes(Definitions): "gwydrau lluosog ym mhobman": "multiple glazing throughout", } + # These are observed data anomalies that we want to ignore + NO_DATA_CASES = [ + "SAP05:Windows", + "Solid, no insulation (assumed)", # A description typically associated with floors, not windows + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()) # In the case of an empty description, we want to return a dictionary with all values set to False # and indicate there was no data - self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description == "SAP05:Windows" + self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description in self.NO_DATA_CASES translation = self.WELSH_TEXT.get(self.description) if translation: From b3053f8518727db1b00c2165b286988dbf5796e9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 16:10:34 +0100 Subject: [PATCH 50/59] handing edge cases for epc cleaning --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 9 ++++++++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 3 +++ etl/epc_clean/epc_attributes/WindowAttributes.py | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index 78ee5f7d..b0105b10 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -127,13 +127,20 @@ class HotWaterAttributes(Definitions): "thermostat, flue gas heat recovery", "ogçör brif system, gydag ynnigçör haul, adfer gwres nwyon ffliw": "from main system, plus solar, flue gas " "heat recovery", + "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder " + "thermostat", } + NODATA_DESCRIPTIONS = [ + "sap05 hot-water", + "sap hot-water" + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()).strip() self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or ( - self.description == "sap05 hot-water" + self.description in self.NODATA_DESCRIPTIONS ) translation = self.WELSH_TEXT.get(self.description) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 430b418d..46cbf52b 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -61,6 +61,7 @@ class MainHeatAttributes(Definitions): "bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage " "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", + "stor wresogyddion trydan": "electric storage heaters", # Should be handled by edge cases ", trydan": ", electric", } diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index f36d445f..75cb8af1 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -19,6 +19,8 @@ class RoofAttributes(Definitions): "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', + "ar oleddf, wedi?i inswleiddio wrth y trawstia": 'pitched, insulated at rafters', + "ar oleddf, wedigçöi inswleiddio wrth y trawstia": 'pitched, insulated at rafters', "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", @@ -35,6 +37,7 @@ class RoofAttributes(Definitions): "ystafell(oedd) to, nenfwd wedigçöi inswleiddio": "roof room(s), ceiling insulated", "ystafell(oedd) to, dim inswleiddio (rhagdybiaeth)": "roof room(s), no insulation (assumed)", "ystafell(oedd) to, dim inswleiddio": "roof room(s), no insulation", + "to gwellt, gydag inswleiddio ychwanegol": "thatched, with additional insulation", } DEFAULT_KEYS = [ diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index a52977e6..8c4d0c45 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -37,6 +37,7 @@ class WindowAttributes(Definitions): NO_DATA_CASES = [ "SAP05:Windows", "Solid, no insulation (assumed)", # A description typically associated with floors, not windows + "Suspended, no insulation (assumed)", # A description typically associated with floors, not windows ] def __init__(self, description: str): From c1b3bc2ecec48a8c99e8eae0fea2a86b42b90f7c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:01:22 +0100 Subject: [PATCH 51/59] debugging descriptions --- backend/app/plan/router.py | 123 ++++++++++++++++++ backend/app/plan/schemas.py | 4 +- .../epc_attributes/HotWaterAttributes.py | 1 + .../MainheatControlAttributes.py | 1 + recommendations/FloorRecommendations.py | 3 +- recommendations/WindowsRecommendations.py | 3 +- 6 files changed, 131 insertions(+), 4 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d1578cc1..3b50d46d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -729,6 +729,129 @@ async def trigger_plan(body: PlanTriggerRequest): scoring_epcs.extend(property_instance.updated_simulation_epcs) recommendations[property_id] = recommendations_with_impact + # For Debugging + recommendation_impact_df = [] + for property_id in recommendations.keys(): + for recs_by_type in recommendations[property_id]: + for rec in recs_by_type: + recommendation_impact_df.append( + { + "property_id": property_id, + "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + "recommendation_id": rec["recommendation_id"], + "type": rec["type"], + "description": rec["description"], + "sap_points": rec["sap_points"], + "co2_equivalent_savings": rec["co2_equivalent_savings"], + "heat_demand": rec["heat_demand"] + } + ) + recommendation_impact_df = pd.DataFrame(recommendation_impact_df) + + surveyed_uprns = [ + 10024087855, 121016117, 121016124, + 10024087902, 121016121, 121016128 + ] + recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] + recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"] == "windows_glazing"] + + actual_impacts_df = pd.DataFrame( + [ + # 10024087855 + {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, + {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, + {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, + {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, + # 121016117 + {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, + {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, + {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, + # 121016124 + {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, + {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, + {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, + # 10024087902 + {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, + {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, + {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, + # 121016121 + {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, + {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, + {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, + # 121016128 + {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, + {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, + ] + ) + + comparison = recommendation_impact_df.merge( + actual_impacts_df, how="inner", on=["uprn", "type"] + ) + + # from utils.s3 import read_dataframe_from_s3_parquet + # training_data = read_dataframe_from_s3_parquet( + # bucket_name="retrofit-data-dev", + # file_key="sap_change_model/2024-08-06-11-19-49/dataset_rooms.parquet" + # ) + # import pickle + # with open("delete_me.pkl", "wb") as f: + # pickle.dump(training_data, f) + + # Read in the pickle + import pickle + with open("delete_me.pkl", "rb") as f: + training_data = pickle.load(f) + + # How do we simulate windows: + ending_cols = [col for col in training_data.columns if col.endswith("_ending")] + starting = {} + for c in ending_cols: + starting_colname = c.replace("_ending", "_starting") + if starting_colname in training_data.columns: + starting[c] = starting_colname + else: + starting[c] = c.replace("_ending", "") + + allowed_to_change = [ + # Windows + "windows_energy_eff_ending", + "glazed_type_ending", + "glazing_type_ending", + "multi_glaze_proportion_ending", + + # Other + "sap_ending", + "heat_demand_ending", + "carbon_ending", + "estimated_perimeter_ending", + "lodgement_year_ending", + "lodgement_month_ending", + "days_to_ending", + "number_habitable_rooms_ending", + "number_heated_rooms_ending", + ] + fixed = [c for c in ending_cols if c not in allowed_to_change + ["uprn"]] + training_fixed = training_data.copy() + for col in fixed: + starting_col = starting[col] + training_fixed = training_fixed[training_fixed[col] == training_fixed[starting_col]] + + training_fixed = training_fixed.reset_index(drop=True) + + # Find the things that change + example = training_fixed.iloc[0] + things_that_change = [] + for c in ending_cols: + if example[c] != example[starting[c]]: + things_that_change.append(c) + # 100051011370 + example[] + # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) scoring_epcs = kwh_client.transform(data=scoring_epcs, cleaned=cleaned) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 68f8bbf5..0ddd9761 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -90,13 +90,13 @@ class PlanTriggerRequest(BaseModel): # Validator to ensure exclusions are within the pre-defined possibilities @validator('exclusions', each_item=True) def check_exclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES: + if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: raise ValueError(f"{v} is not an allowed exclusion") return v @validator('inclusions', each_item=True) def check_inclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES: + if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: raise ValueError(f"{v} is not an allowed inclusion") return v diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index b0105b10..76b4e6fa 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -129,6 +129,7 @@ class HotWaterAttributes(Definitions): "heat recovery", "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder " "thermostat", + "o r brif system, gydag ynni r haul": "from main system, plus solar", } NODATA_DESCRIPTIONS = [ diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 2759096d..eaa701da 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -115,6 +115,7 @@ class MainheatControlAttributes(Definitions): 't+ól un gyfradd, trvs': 'single rate heating, trvs', 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', + 'rheoli r tal a llaw': 'manual charge control', } def __init__(self, description: str): diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index a1f63f96..db18a458 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -68,7 +68,8 @@ class FloorRecommendations(Definitions): measures = MEASURE_MAP["floor_insulation"] if measures is None else measures - if not measures: + # If we have no measures or none of the measures are relevant, we can't recommend anything + if not measures or not any(x in measures for x in MEASURE_MAP["floor_insulation"]): return u_value = self.property.floor["thermal_transmittance"] diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index ae7f7057..bc91f801 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -60,7 +60,8 @@ class WindowsRecommendations: return if windows_area is not None: - raise Exception("We have windows area, we should use this data for our recommendations!!!") + # TODO - we don't have a price for this so we can't recommend it + print("We have windows area, we should use this data for our recommendations!!!") # We scale the number of windows based on the proportion of existing glazing if self.property.data["multi-glaze-proportion"] != "": From 6a45789edd94aab73e9fa55c903e864c67f6d105 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:47:53 +0100 Subject: [PATCH 52/59] investigating secondary glazing recommendations --- backend/Property.py | 5 +-- backend/app/plan/router.py | 44 ++++++++++++++++--- .../epc_attributes/RoofAttributes.py | 1 + .../epc_attributes/WindowAttributes.py | 1 + 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 0d194a79..77415d0e 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -503,11 +503,10 @@ class Property: output["lighting_energy_eff_ending"] = "Very Good" if recommendation["type"] == "windows_glazing": + is_secondary_glazing = recommendation["is_secondary_glazing"] output["multi_glaze_proportion_ending"] = 100 if output["windows_energy_eff_ending"] not in ["Average", "Good", "Very Good"]: - output["windows_energy_eff_ending"] = "Average" - - is_secondary_glazing = recommendation["is_secondary_glazing"] + output["windows_energy_eff_ending"] = "Average" if not is_secondary_glazing else "Good" if output["glazing_type_ending"] == "multiple": pass diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3b50d46d..05c79a22 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -843,14 +843,44 @@ async def trigger_plan(body: PlanTriggerRequest): training_fixed = training_fixed.reset_index(drop=True) + # Get the recommendation config for this uprn + uprn = 121016121 + property_instance = [p for p in input_properties if p.uprn == uprn][0] + property_recs = recommendations[property_instance.id] + window_recs = [r for r in property_recs if r[0]["type"] == "windows_glazing"][0] + window_recs[0].keys() + window_recs[0]["description_simulation"]["multi-glaze-proportion"] + # TODO: - In description_simulation for windows, we update glazed-type but in the model training data there + # is a column called "glazing-type". + # - We don't update glazed-area (should be "Much More Than Typical" most likely? Or Normal??) + # TODO: I think we update eveything that we actually need to, when simulating the recommendation impact for the + # ML models + # TODO: Secondary glazing appears to go to "Good", not "Average". Investigate why + # TODO: For the two properties, force recommendations for double glazing and check impact + + z = training_data[training_data["glazed_type_ending"] == "secondary glazing"] + z = z[z["multi_glaze_proportion_ending"] == 100] + z["windows_energy_eff_ending"].value_counts() + # Find the things that change - example = training_fixed.iloc[0] - things_that_change = [] - for c in ending_cols: - if example[c] != example[starting[c]]: - things_that_change.append(c) - # 100051011370 - example[] + example = training_fixed.iloc[3] + for _, example in training_fixed.iterrows(): + things_that_change = [] + for c in ending_cols: + if example[c] != example[starting[c]]: + things_that_change.append(c) + if len(things_that_change) > 4: + print(things_that_change) + print(example["uprn"]) + # blah + + # 100051011370 (doesn't change in actual glazing) + # example["glazed_type_ending"] + # double glazing installed before 2002 + # example["glazed_type_starting"] + # double glazing, unknown install date + + # 100040925015 # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 75cb8af1..0fc2156e 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -15,6 +15,7 @@ class RoofAttributes(Definitions): "ar oleddf, wedi?i inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", + "ar oleddf, wedi?i inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index 8c4d0c45..2b1dc172 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -27,6 +27,7 @@ class WindowAttributes(Definitions): "gwydrau triphlyg llawn": "fully triple glazed", "gwydrau triphlyg rhannol": "partial triple glazed", "gwydrau triphlyg mwyaf": "mostly triple glazed", + "gwydrau triphlyg gan mwyaf": "mostly triple glazed", "gwydrau eilaidd llawn": "full secondary glazing", "gwydrau eilaidd mwyaf": "mostly secondary glazing", "gwydrau eilaidd rhannol": "partial secondary glazing", From 957d006cb8d7e43e4f73a6ccf89021840da05b99 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:59:44 +0100 Subject: [PATCH 53/59] fixing cleaning of welsh descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 46cbf52b..896f73b9 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -37,6 +37,7 @@ class MainHeatAttributes(Definitions): "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan, pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, " "trydan": "air source heat pump, underfloor heating, electric", "cynllun cymunedol": "community scheme", + "cynllun cymunedol, heat from boilers - mains gas": "community scheme", "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas", "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs', "bwyler a rheiddiaduron, tanwydd di-fwg": "boiler and radiators, smokeless fuel", diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index eaa701da..d7509d47 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -116,6 +116,7 @@ class MainheatControlAttributes(Definitions): 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', 'rheoli r tal a llaw': 'manual charge control', + 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only' } def __init__(self, description: str): From 0e8136d445f0f84c1b56b2611853aa1c3b157b7c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 27 Sep 2024 16:18:31 +0100 Subject: [PATCH 54/59] debugging epc cleaning --- etl/epc_clean/epc_attributes/FloorAttributes.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 8 ++++++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 3 ++- etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 9d0f514d..bba33424 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -32,7 +32,8 @@ class FloorAttributes(Definitions): "solet, wedigçöi inswleiddio": "solid, insulated", "solet, wedi???i inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", - "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation" + "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", + "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation", } def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 896f73b9..ea61c3b4 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -63,6 +63,13 @@ class MainHeatAttributes(Definitions): "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", "stor wresogyddion trydan": "electric storage heaters", + # Not 100% certain - the translation is "bottled gas" + "bwyler a rheiddiaduron, nwy potel": "boiler and radiators, lpg", + "gwresogyddion trydan cludadwy wedi i ragdybio ar gyfer y rhan fwyaf o r ystafelloedd": "portable electric " + "heaters assumed for " + "most rooms", + "st r wresogyddion trydan": "electric storage heaters", + "dim system ar gael, rhagdybir bod gwresogyddion trydan, trydan": "no system present, electric heaters assumed", # Should be handled by edge cases ", trydan": ", electric", } @@ -77,6 +84,7 @@ class MainHeatAttributes(Definitions): "radiator heating, electric": "room heaters, electric", "hot-water-only systems, gas": "no system present, electric heaters assumed", "gas-fired heat pumps, electric": "air source heat pump, electric", + "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index d7509d47..4a846498 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -116,7 +116,8 @@ class MainheatControlAttributes(Definitions): 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', 'rheoli r tal a llaw': 'manual charge control', - 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only' + 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only', + "rheoli'r t l llaw": "manual charge control", } def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 0fc2156e..2eacc951 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -27,6 +27,7 @@ class RoofAttributes(Definitions): "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", "yn wastad, dim inswleiddio": "flat, no insulation", "yn wastad, wedigçöi inswleiddio (rhagdybiaeth)": "flat, insulated (assumed)", + "yn wastad, wedi?i hinswleiddio (rhagdybiaeth)": "flat, insulated (assumed)", "yn wastad, wedigçöi inswleiddio": "flat, insulated", "(eiddo arall uwchben)": "(another dwelling above)", "(annedd arall uwchben)": "(another dwelling above)", From 32d702e9306614f29d2d4a1268e6706bf99b9314 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 27 Sep 2024 17:10:04 +0100 Subject: [PATCH 55/59] moved leds recommendations earlier --- backend/app/plan/router.py | 11 ++++++++++- recommendations/Recommendations.py | 12 ++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 05c79a22..90353052 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -738,6 +738,7 @@ async def trigger_plan(body: PlanTriggerRequest): { "property_id": property_id, "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + "address": [p.address for p in input_properties if p.id == property_id][0], "recommendation_id": rec["recommendation_id"], "type": rec["type"], "description": rec["description"], @@ -753,7 +754,9 @@ async def trigger_plan(body: PlanTriggerRequest): 10024087902, 121016121, 121016128 ] recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"] == "windows_glazing"] + # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( + # ["windows_glazing", "internal_wall_insulation"]) + # ] actual_impacts_df = pd.DataFrame( [ @@ -793,6 +796,12 @@ async def trigger_plan(body: PlanTriggerRequest): actual_impacts_df, how="inner", on=["uprn", "type"] ) + property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] + property = [p for p in input_properties if p.uprn == 121016128][0] + print(property.data["current-energy-efficiency"]) + print(property_recs["sap_points"].sum()) + property_recs["address"] + # from utils.s3 import read_dataframe_from_s3_parquet # training_data = read_dataframe_from_s3_parquet( # bucket_name="retrofit-data-dev", diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 5037f450..d5e37f8e 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -157,6 +157,12 @@ class Recommendations: property_recommendations.append(self.floor_recommender.recommendations) phase += 1 + if "low_energy_lighting" in measures: + self.lighting_recommender.recommend(phase=phase) + if self.lighting_recommender.recommendation: + property_recommendations.append(self.lighting_recommender.recommendation) + phase += 1 + if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation self.windows_recommender.recommend(phase=phase) @@ -233,12 +239,6 @@ class Recommendations: property_recommendations.append(self.hotwater_recommender.recommendations) phase += 1 - if "low_energy_lighting" in measures: - self.lighting_recommender.recommend(phase=phase) - if self.lighting_recommender.recommendation: - property_recommendations.append(self.lighting_recommender.recommendation) - phase += 1 - if "secondary_heating" in measures: self.secondary_heating_recommender.recommend(phase=phase) if self.secondary_heating_recommender.recommendation: From fb3fef5a4a6fdad9a39e665c46d931d9bf401bb9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 28 Sep 2024 17:03:41 +0100 Subject: [PATCH 56/59] reviewing model performance for vectis - complete --- backend/app/plan/router.py | 143 +++++++++++---------- backend/app/plan/schemas.py | 4 +- recommendations/FloorRecommendations.py | 4 +- recommendations/LightingRecommendations.py | 33 +++++ recommendations/Recommendations.py | 19 ++- recommendations/WindowsRecommendations.py | 21 ++- 6 files changed, 142 insertions(+), 82 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 90353052..6e4d8475 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -730,77 +730,80 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations[property_id] = recommendations_with_impact # For Debugging - recommendation_impact_df = [] - for property_id in recommendations.keys(): - for recs_by_type in recommendations[property_id]: - for rec in recs_by_type: - recommendation_impact_df.append( - { - "property_id": property_id, - "uprn": [p.uprn for p in input_properties if p.id == property_id][0], - "address": [p.address for p in input_properties if p.id == property_id][0], - "recommendation_id": rec["recommendation_id"], - "type": rec["type"], - "description": rec["description"], - "sap_points": rec["sap_points"], - "co2_equivalent_savings": rec["co2_equivalent_savings"], - "heat_demand": rec["heat_demand"] - } - ) - recommendation_impact_df = pd.DataFrame(recommendation_impact_df) - - surveyed_uprns = [ - 10024087855, 121016117, 121016124, - 10024087902, 121016121, 121016128 - ] - recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( - # ["windows_glazing", "internal_wall_insulation"]) + # recommendation_impact_df = [] + # for property_id in recommendations.keys(): + # for recs_by_type in recommendations[property_id]: + # for rec in recs_by_type: + # recommendation_impact_df.append( + # { + # "property_id": property_id, + # "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + # "address": [p.address for p in input_properties if p.id == property_id][0], + # "recommendation_id": rec["recommendation_id"], + # "type": rec["type"], + # "description": rec["description"], + # "sap_points": rec["sap_points"], + # "co2_equivalent_savings": rec["co2_equivalent_savings"], + # "heat_demand": rec["heat_demand"] + # } + # ) + # recommendation_impact_df = pd.DataFrame(recommendation_impact_df) + # + # surveyed_uprns = [ + # 10024087855, 121016117, 121016124, + # 10024087902, 121016121, 121016128 # ] - - actual_impacts_df = pd.DataFrame( - [ - # 10024087855 - {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, - {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, - {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, - {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, - # 121016117 - {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, - {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, - {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, - # 121016124 - {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, - {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, - {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, - # 10024087902 - {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, - {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, - {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, - # 121016121 - {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, - {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, - {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, - # 121016128 - {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, - {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, - ] - ) - - comparison = recommendation_impact_df.merge( - actual_impacts_df, how="inner", on=["uprn", "type"] - ) - - property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] - property = [p for p in input_properties if p.uprn == 121016128][0] - print(property.data["current-energy-efficiency"]) - print(property_recs["sap_points"].sum()) - property_recs["address"] + # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] + # # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( + # # ["windows_glazing", "internal_wall_insulation"]) + # # ] + # + # actual_impacts_df = pd.DataFrame( + # [ + # # 10024087855 + # {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, + # {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, + # {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, + # {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, + # # 121016117 + # {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, + # {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, + # {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, + # # 121016124 + # {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, + # {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, + # {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, + # # 10024087902 + # {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, + # {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, + # {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, + # # 121016121 + # {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, + # {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, + # {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, + # # 121016128 + # {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, + # {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, + # ] + # ) + # + # comparison = recommendation_impact_df.merge( + # actual_impacts_df, how="inner", on=["uprn", "type"] + # ) + # + # print(recommendation_impact_df.groupby(["uprn"])["sap_points"].sum()) + # property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] + # property = [p for p in input_properties if p.uprn == 121016128][0] + # print(property.data["current-energy-efficiency"]) + # print(property_recs["sap_points"].sum()) + # print(property_recs["type"]) + # print(float(property.data["current-energy-efficiency"]) + property_recs["sap_points"].sum()) + # recommendations[property.id][2][0]["simulation_config"] # from utils.s3 import read_dataframe_from_s3_parquet # training_data = read_dataframe_from_s3_parquet( diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 0ddd9761..c08cdefc 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -36,7 +36,8 @@ SPECIFIC_MEASURES = [ # Solar "solar_pv", # Windows Glazing - "windows", + "double_glazing", + "secondary_glazing", # Mechanical ventilation "ventilation", # Other @@ -62,6 +63,7 @@ MEASURE_MAP = { "roof_insulation": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"], "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"], "heating": ["boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"], + "windows": ["double_glazing", "secondary_glazing"], } diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index db18a458..d82162da 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -224,7 +224,9 @@ class FloorRecommendations(Definitions): simulation_config = { **floor_simulation_config, - "floor_thermal_transmittance_ending": new_u_value, + # We don't simulate the impact using this U-value, but rather the average because this + # variable is way too volatile. Will likely be removed from the model + "floor_thermal_transmittance_ending": 0.685593, } self.recommendations.append( diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 92394c11..2b0e8724 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -1,3 +1,5 @@ +import pandas as pd + from backend.Property import Property from typing import List from recommendations.Costs import Costs @@ -30,6 +32,37 @@ class LightingRecommendations: self.material = material[0] self.recommendation = [] + @classmethod + def get_sap_limit(cls, lighting_energy_efficiency: str, lighting_proportion: float): + """ + Lighting seems to be a more straight forward measure to estimate SAP points for, based on the starting + energy efficiency rating. + + We seem to have the following brackes based on % of LEDs in outlets + Very poor: 0 - 9% + Poor: 10 - 24% + Average: 25 - 44% + Good: 45 - 69% + Very good: 70 - 100% + :return: + """ + + if lighting_energy_efficiency == "Very Good": + return 0 + + if lighting_energy_efficiency in ["Good", "Average"]: + return cls.SAP_LOWER_LIMIT + + # If lighting_energy_efficiency is missing, we'll use the proportion of low energy lighting + if not lighting_energy_efficiency or pd.isnull(lighting_energy_efficiency): + if lighting_proportion >= 0.7: + return 0 + if lighting_proportion >= 0.25: + return cls.SAP_LOWER_LIMIT + return cls.SAP_LIMIT + + return cls.SAP_LIMIT + @staticmethod def estimate_lighting_impact(number_of_bulbs: int): """ diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index d5e37f8e..526cb2a2 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -80,6 +80,13 @@ class Recommendations: inclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.inclusions] exclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.exclusions] + # We need to unlist any lists, but we should check if they're lists first + inclusions_full = [ + item for sublist in inclusions_full for item in (sublist if isinstance(sublist, list) else [sublist]) + ] + exclusions_full = [ + item for sublist in exclusions_full for item in (sublist if isinstance(sublist, list) else [sublist]) + ] # If inclusions and exclusions are empty, it means that nothing was specified, so we allow # all recommendation types @@ -163,9 +170,9 @@ class Recommendations: property_recommendations.append(self.lighting_recommender.recommendation) phase += 1 - if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: + if "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation - self.windows_recommender.recommend(phase=phase) + self.windows_recommender.recommend(phase=phase, measures=measures) if self.windows_recommender.recommendation: property_recommendations.append(self.windows_recommender.recommendation) phase += 1 @@ -538,11 +545,11 @@ class Recommendations: # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2 if rec["type"] == "low_energy_lighting": + lighting_sap_limit = LightingRecommendations.get_sap_limit( + property_instance.data["lighting-energy-eff"], + property_instance.lighting["low_energy_proportion"] + ) - if property_instance.data["low-energy-lighting"] < 50: - lighting_sap_limit = LightingRecommendations.SAP_LIMIT - else: - lighting_sap_limit = LightingRecommendations.SAP_LOWER_LIMIT property_phase_impact["sap"] = min(property_phase_impact["sap"], lighting_sap_limit) property_phase_impact["carbon"] = min( property_phase_impact["carbon"], rec["co2_equivalent_savings"] diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index bc91f801..235d9ee2 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -3,6 +3,7 @@ from typing import List import numpy as np from backend.Property import Property +from backend.app.plan.schemas import MEASURE_MAP from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes from recommendations.Costs import Costs from recommendations.recommendation_utils import override_costs, check_simulation_difference @@ -32,7 +33,7 @@ class WindowsRecommendations: raise ValueError("There should only be one window glazing material") self.glazing_material = self.glazing_material[0] - def recommend(self, phase=0): + def recommend(self, measures=None, phase=0): """ This method will recommend the best possible glazing options for a property. @@ -41,14 +42,26 @@ class WindowsRecommendations: :return: """ + measures = MEASURE_MAP["windows"] if measures is None else measures + + # If we have no windows recs, leave + if not any(x in measures for x in MEASURE_MAP["windows"]): + return + # If the property is in a conservation area or is a listed building, it becomes more difficult to install # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it # requires planning permission and might require a more expensive window type, such as timber. number_of_windows = self.property.number_of_windows - is_secondary_glazing = self.property.restricted_measures or ( - self.property.windows["glazing_type"] == "secondary" - ) + + if "double_glazing" in measures and "secondary_glazing" not in measures: + is_secondary_glazing = False + elif "secondary_glazing" in measures and "double_glazing" not in measures: + is_secondary_glazing = True + else: + is_secondary_glazing = self.property.restricted_measures or ( + self.property.windows["glazing_type"] == "secondary" + ) windows_area = self.property.windows_area if not number_of_windows: From ceb003ec7a82b954c3c8b5f43292a4726b042e1e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 11:02:32 +0100 Subject: [PATCH 57/59] debugging epc descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index ea61c3b4..52dc1bc7 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -166,6 +166,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == ', mains gas': + self.edge_case_result['has_mains_gas'] = True + self.is_edge_case = True + return + if self.description == 'community, community': self.edge_case_result['has_community_scheme'] = True self.is_edge_case = True From 591b9522251bcf138357b96128ff4cdebaa5607a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 11:53:04 +0100 Subject: [PATCH 58/59] adding the assessment information to retrieve_find_my_epc_data --- etl/bill_savings/data_collection.py | 42 +++++++++++++++++-- etl/customers/aiha/epc_surveyor_list.py | 41 ++++++++++++++++++ .../MainheatControlAttributes.py | 7 +++- 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 etl/customers/aiha/epc_surveyor_list.py diff --git a/etl/bill_savings/data_collection.py b/etl/bill_savings/data_collection.py index 49bcff82..ee8a228f 100644 --- a/etl/bill_savings/data_collection.py +++ b/etl/bill_savings/data_collection.py @@ -100,9 +100,44 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e bills = address_res.find('div', {'id': 'bills-affected'}) bills_list = bills.find_all('li') if not bills_list: - return None - heating_text = bills_list[0].text - hot_water_text = bills_list[1].text + # If this is the case, it's usually becaue the EPC was very old. Early EPCs did not have this information + heating_text = None + hot_water_text = None + else: + heating_text = bills_list[0].text + hot_water_text = bills_list[1].text + + # Search for the assessment informaton + assessment_information = address_res.find('div', {'id': 'information'}) + # Parse this information + rows = assessment_information.find_all('div', class_='govuk-summary-list__row') + # Create a dictionary to hold the parsed information + assessment_data = {} + for row in rows: + key = row.find('dt').text.strip() + if key == "Type of assessment": + # We dont reliably extract this + continue + value_tag = row.find('dd') + + # Check if value contains a link (email) + if value_tag.find('a'): + value = value_tag.find('a').text.strip() + elif value_tag.find('summary'): + value = value_tag.find('span').text.strip() + else: + value = value_tag.text.strip() + + assessment_data[key] = value + + expected_keys = [ + 'Assessor’s name', 'Telephone', 'Email', 'Accreditation scheme', 'Assessor’s ID', 'Assessor’s declaration', + 'Date of assessment', 'Date of certificate' + ] + # Check we have all the expected keys + for key in expected_keys: + if key not in assessment_data: + raise ValueError(f"Missing key: {key}") resulting_data = { 'extracted_uprn': uprn, @@ -114,6 +149,7 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e "potential_epc_efficiency": int(potential_rating.split(' ')[-1]), "heating_text": heating_text, "hot_water_text": hot_water_text, + **assessment_data } return resulting_data diff --git a/etl/customers/aiha/epc_surveyor_list.py b/etl/customers/aiha/epc_surveyor_list.py new file mode 100644 index 00000000..b85139ae --- /dev/null +++ b/etl/customers/aiha/epc_surveyor_list.py @@ -0,0 +1,41 @@ +import pandas as pd +import numpy as np +import time +from tqdm import tqdm +from etl.bill_savings.data_collection import retrieve_find_my_epc_data, calculate_expiry_date + + +def main(): + """ + This script handles pulling the surveyor names and acreditation details for Surveyors who have completed + the newest EPC for AIHA's properties + """ + + epc_data = pd.read_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv") + epc_data = epc_data[["uprn", "address", "address1", "postcode", "lodgement-date"]] + + epc_collected_data = [] + for _, unit in tqdm(epc_data.iterrows(), total=len(epc_data)): + time.sleep(np.random.uniform(0.2, 1.5)) + uprn = int(unit["uprn"]) + address = unit["address1"] + postcode = unit["postcode"] + expected_expiry_date = calculate_expiry_date(unit["lodgement-date"]) + + response = retrieve_find_my_epc_data( + uprn=uprn, + postcode=postcode, + address=address, + expected_expiry_date=expected_expiry_date + ) + if response is None: + raise Exception("fix me") + epc_collected_data.append(response) + + epc_collected_data = pd.DataFrame(epc_collected_data) + + for x in epc_collected_data: + keys = x.keys() + # Check for None keys + if any(k is None for k in keys): + frew diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 4a846498..a13823d2 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -120,10 +120,15 @@ class MainheatControlAttributes(Definitions): "rheoli'r t l llaw": "manual charge control", } + NO_DATA_DESCRIPTIONS = [ + "SAP05:Main-Heating-Controls", + "SAP:Main-Heating-Controls", + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()).strip() self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or ( - description == "SAP05:Main-Heating-Controls" + description in self.NO_DATA_DESCRIPTIONS ) translation = self.WELSH_TEXT.get(self.description) From 986174b95d3aef7ad77650dc995df00435470872 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 12:57:42 +0100 Subject: [PATCH 59/59] handling new EPC description --- etl/customers/aiha/epc_surveyor_list.py | 31 ++++++++++++++++--- .../epc_attributes/MainheatAttributes.py | 1 + .../test_mainheat_attributes_cases.py | 17 +++++++++- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/etl/customers/aiha/epc_surveyor_list.py b/etl/customers/aiha/epc_surveyor_list.py index b85139ae..cec72928 100644 --- a/etl/customers/aiha/epc_surveyor_list.py +++ b/etl/customers/aiha/epc_surveyor_list.py @@ -34,8 +34,29 @@ def main(): epc_collected_data = pd.DataFrame(epc_collected_data) - for x in epc_collected_data: - keys = x.keys() - # Check for None keys - if any(k is None for k in keys): - frew + epc_collected_data = epc_data[["uprn", "address", "address1", "postcode"]].merge( + epc_collected_data, left_on="uprn", right_on="extracted_uprn" + ) + + elmhurst_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["NHER", "Stroma Certification Ltd", "Elmhurst Energy Systems Ltd"] + ) + ] + + quidos_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["Quidos Limited"] + ) + ] + + ecmk_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["ECMK"] + ) + ] + + # Store the data: + elmhurst_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Elmhurst Surveys.csv") + quidos_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Quidos Surveys.csv") + ecmk_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/ECMK Surveys.csv") diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 52dc1bc7..051db8c2 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -85,6 +85,7 @@ class MainHeatAttributes(Definitions): "hot-water-only systems, gas": "no system present, electric heaters assumed", "gas-fired heat pumps, electric": "air source heat pump, electric", "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas", + "heat pump, warm air, mains gas": "air source heat pump, warm air, mains gas", } edge_case_result = {} diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 82675a74..51fc3416 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1692,6 +1692,21 @@ mainheat_cases = [ 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False - } + }, + { + "original_description": "heat pump, warm air, mains gas", # This gets remapped to air source heat pump + 'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': True, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': True, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, + 'has_underfloor_heating': False + } ]