From 53e68f8d76b3f8146166ba575c7507908e3b1643 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 15:26:38 +0100 Subject: [PATCH] added tests for final property --- etl/sfr/midlands_portfolio_est_funding.py | 100 ++++++--- recommendations/HeatingRecommender.py | 19 +- .../test_data/heating_recommendations_data.py | 209 +++++++++++------- 3 files changed, 218 insertions(+), 110 deletions(-) diff --git a/etl/sfr/midlands_portfolio_est_funding.py b/etl/sfr/midlands_portfolio_est_funding.py index 09102cfb..017fd223 100644 --- a/etl/sfr/midlands_portfolio_est_funding.py +++ b/etl/sfr/midlands_portfolio_est_funding.py @@ -57,10 +57,18 @@ def app(): ['none', "below average"] ) + epc_data["needs_solid_wall"] = (epc_data["is_solid_brick"] | epc_data["is_system_built"]) & epc_data[ + "insulation_thickness_wall"].isin(['none', "below average"]) + + epc_data["could_take_solar"] = (epc_data["is_flat"] | epc_data["is_pitched"]) + loft_insulation_per_m2 = 16.07 flat_roof_insulation_per_m2 = 195 cwi_per_m2 = 14.21 + ewi_per_m2 = 200 gbis_abs = 30 + eco4_abs = 24 + solar_pv_cost = 4009 # We assume the work will take the home from a high D to a low D def get_abs(floor_area): @@ -75,6 +83,19 @@ def app(): return 350.1 + # We assume the work will take the home from a high E to a high C + def get_eco4_abs(floor_area): + if floor_area <= 72: + return 596.6 + + if floor_area <= 97: + return 650.2 + + if floor_area <= 199: + return 755.8 + + return 1347.1 + estimated_costs = [] for _, home in epc_data.iterrows(): to_append = { @@ -89,30 +110,59 @@ def app(): n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"]) floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5 - # Check if it needs the walls done - if home["needs_cavity_done"]: - # We estimate the amount of insulation required - est_perimeter = estimate_perimeter( - floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, - num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors - ) + # We estimate the amount of insulation required + est_perimeter = estimate_perimeter( + floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, + num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors + ) - insulation_needed = estimate_external_wall_area( - num_floors=n_floors, - floor_height=floor_height, - perimeter=est_perimeter, - built_form=home["BUILT_FORM"], - ) - cost_of_insulation = insulation_needed * cwi_per_m2 + insulation_needed = estimate_external_wall_area( + num_floors=n_floors, + floor_height=floor_height, + perimeter=est_perimeter, + built_form=home["BUILT_FORM"], + ) - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + # At the very least we'll need solid wall + solar + if home["needs_solid_wall"] and home["could_take_solar"]: + measure = "EWI + Solar" + + total_cost = insulation_needed * ewi_per_m2 + solar_pv_cost + + eco4_project_abs = get_eco4_abs(home["TOTAL_FLOOR_AREA"]) + eco4_available_funding = eco4_project_abs * eco4_abs + + cost_of_work_after_funding = total_cost - eco4_available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "eco4", + "available_funding": eco4_available_funding, + "measure": measure, + "project_abs": eco4_project_abs, + "cost_of_work": total_cost, + "cost_of_work_after_funding": cost_of_work_after_funding, + } + + estimated_costs.append(to_append) + continue + + # Check if it needs the walls done + if home["needs_cavity_done"]: + cost_of_insulation = insulation_needed * cwi_per_m2 + + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding + + to_append = { + **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": "Cavity Wall Insulation", - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -130,14 +180,17 @@ def app(): roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors cost_of_insulation = roof_area * flat_roof_insulation_per_m2 - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": measure, - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -145,13 +198,10 @@ def app(): estimated_costs = pd.DataFrame(estimated_costs) - estimated_costs.groupby("measure")["available_funding"].mean() - estimated_costs["measure"].value_counts() - estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv") - epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") + # epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( + # "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") n_properties_for_ashp = epc_data[ (epc_data["PROPERTY_TYPE"] == "House") & diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index e40c1736..fea2d8db 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -91,7 +91,7 @@ class HeatingRecommender: # If the property already has room heaters then we recommend HHR as an option since the home already has # a variation of room heaters - + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( @@ -130,6 +130,13 @@ class HeatingRecommender: not self.property.main_heating["has_mains_gas"] and self.property.data["mains-gas-flag"] ) + # Additionally, if the property has a gas connection, is using gas heating but doesn't have a boiler, + # we recommend a boiler + non_boiler_gas_heating = ( + self.property.data["mains-gas-flag"] and + self.property.main_heating["has_mains_gas"] and + not self.property.main_heating["has_boiler"] + ) is_valid = ( ( @@ -138,7 +145,8 @@ class HeatingRecommender: electic_heating_has_mains or has_room_heaters or portable_heaters_has_mains or - non_gas_boiler + non_gas_boiler or + non_boiler_gas_heating ) and (not ashp_only_heating_recommendation) and ("boiler_upgrade" in measures) and @@ -842,12 +850,13 @@ class HeatingRecommender: has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] - has_inefficient_mains_water = ( - self.property.hotwater["clean_description"] in ["From main system"] and + # We check if there's a mains connection and the hot water is inefficient, as this will improve with a boiler + has_inefficient_water = ( + self.property.data["mains-gas-flag"] and self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] ) - if has_inefficient_space_heating or has_inefficient_mains_water: + if has_inefficient_space_heating or has_inefficient_water: boiler_size = self.estimate_boiler_size( property_type=self.property.data["property-type"], built_form=self.property.data["built-form"], diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index fea53e2b..e6225299 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1364,86 +1364,135 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" "connection so we recommend a gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '272170070262009042917361440218801', 'address1': '52, Chiswick Walk', 'address2': None, + 'address3': None, 'postcode': 'B37 6TA', 'building-reference-number': 479790668, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 31, + 'potential-energy-efficiency': 50, 'property-type': 'Flat', 'built-form': 'End-Terrace', + 'inspection-date': '2009-04-29', 'local-authority': 'E08000029', 'constituency': 'E14000812', + 'county': None, + 'lodgement-date': '2009-04-29', 'transaction-type': 'marketed sale', 'environment-impact-current': 37, + 'environment-impact-potential': 42, 'energy-consumption-current': 548, 'energy-consumption-potential': 459, + 'co2-emissions-current': 5.8, 'co2-emiss-curr-per-floor-area': 89, 'co2-emissions-potential': 5.0, + 'lighting-cost-current': 60, 'lighting-cost-potential': 30, 'heating-cost-current': 751, + 'heating-cost-potential': 601, 'hot-water-cost-current': 239, 'hot-water-cost-potential': 129, + 'total-floor-area': 65.04, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': '1st', + 'flat-top-storey': 'Y', 'flat-storey-count': 1.0, 'main-heating-controls': 2504.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 2, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'To external air, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', + 'windows-env-eff': 'Good', 'walls-description': 'System built, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Portable electric heaters', + 'roof-description': 'Pitched, limited insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Warm air, mains gas', 'mainheat-energy-eff': 'Good', + 'mainheat-env-eff': 'Good', 'mainheatcont-description': 'Programmer and room thermostat', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'unheated corridor', 'unheated-corridor-length': 5.63, + 'floor-height': 2.32, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '52, Chiswick Walk', 'local-authority-label': 'Solihull', + 'constituency-label': 'Meriden', 'posttown': 'BIRMINGHAM', + 'construction-age-band': 'England and Wales: 1967-1975', 'lodgement-datetime': '2009-04-29 17:36:14', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, + 'uprn': 100070955137, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has warm air mains gas heating, so we recommend a gas condensing boiler" } ] -import random -from pathlib import Path -import inspect -import pandas as pd - -# this can be used to get example data to build the test cases -src_file_path = inspect.getfile(lambda: None) -EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" -epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] -directory = random.sample(epc_directories, 1)[0] -data = pd.read_csv(directory / "certificates.csv", low_memory=False) -# Rename the columns to the same format as the api returns -data.columns = [c.replace("_", "-").lower() for c in data.columns] -data["floor-height"] = data["floor-height"].fillna(2.45) - -used_examples = pd.DataFrame( - [ - { - "mainheat-description": x["epc"]["mainheat-description"], - "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], - "property-type": x["epc"]["property-type"], - "built-form": x["epc"]["built-form"], - "used": True - } for x in testing_examples - ] -) - -data = data.merge( - used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] -) -data = data[pd.isnull(data["used"])].drop(columns=["used"]) - -eg = data.sample(1).to_dict("records")[0] -print(eg["mainheat-description"]) -print(eg["mainheat-energy-eff"]) -print(eg["property-type"]) -print(eg["built-form"]) -print(eg["mainheatcont-description"]) - -### We also use the Midlands EPC F/G portfolio to get examples to create tests - -completed_descriptions = [ - "Portable electric heaters assumed for most rooms", - "Boiler and radiators, oil", - "Boiler and radiators, mains gas", - "Room heaters, mains gas", - "No system present: electric heaters assumed", - "Room heaters, electric", - "Electric storage heaters", - "Boiler and radiators, LPG", - "Boiler and radiators, electric", - "Boiler and radiators, dual fuel (mineral and wood)", - "Boiler and radiators, coal", - "Boiler and radiators, smokeless fuel", - "Boiler and radiators, wood pellets", - "Room heaters, dual fuel (mineral and wood)", - "Air source heat pump, radiators, electric", - "Portable electric heaters assumed for most rooms, Room heaters, electric", - "Boiler and radiators, mains gas, Electric storage heaters", - "Room heaters, anthracite", - "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", - "Electric underfloor heating", -] - -portfolio = pd.read_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" -) -portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] -portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -portfolio['sheating-energy-eff'] = None -portfolio['sheating-env-eff'] = None -portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) - -print(portfolio["mainheat-description"].value_counts()) - -eg = portfolio[ - (portfolio["mainheat-description"] == "Warm air, Electricaire") -].sample(1) -eg = eg.squeeze().to_dict() -print(eg) +# import random +# from pathlib import Path +# import inspect +# import pandas as pd +# +# # this can be used to get example data to build the test cases +# src_file_path = inspect.getfile(lambda: None) +# EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" +# epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] +# directory = random.sample(epc_directories, 1)[0] +# data = pd.read_csv(directory / "certificates.csv", low_memory=False) +# # Rename the columns to the same format as the api returns +# data.columns = [c.replace("_", "-").lower() for c in data.columns] +# data["floor-height"] = data["floor-height"].fillna(2.45) +# +# used_examples = pd.DataFrame( +# [ +# { +# "mainheat-description": x["epc"]["mainheat-description"], +# "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], +# "property-type": x["epc"]["property-type"], +# "built-form": x["epc"]["built-form"], +# "used": True +# } for x in testing_examples +# ] +# ) +# +# data = data.merge( +# used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] +# ) +# data = data[pd.isnull(data["used"])].drop(columns=["used"]) +# +# eg = data.sample(1).to_dict("records")[0] +# print(eg["mainheat-description"]) +# print(eg["mainheat-energy-eff"]) +# print(eg["property-type"]) +# print(eg["built-form"]) +# print(eg["mainheatcont-description"]) +# +# ### We also use the Midlands EPC F/G portfolio to get examples to create tests +# +# completed_descriptions = [ +# "Portable electric heaters assumed for most rooms", +# "Boiler and radiators, oil", +# "Boiler and radiators, mains gas", +# "Room heaters, mains gas", +# "No system present: electric heaters assumed", +# "Room heaters, electric", +# "Electric storage heaters", +# "Boiler and radiators, LPG", +# "Boiler and radiators, electric", +# "Boiler and radiators, dual fuel (mineral and wood)", +# "Boiler and radiators, coal", +# "Boiler and radiators, smokeless fuel", +# "Boiler and radiators, wood pellets", +# "Room heaters, dual fuel (mineral and wood)", +# "Air source heat pump, radiators, electric", +# "Portable electric heaters assumed for most rooms, Room heaters, electric", +# "Boiler and radiators, mains gas, Electric storage heaters", +# "Room heaters, anthracite", +# "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", +# "Electric underfloor heating", +# "Warm air, Electricaire" +# ] +# +# portfolio = pd.read_excel( +# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" +# ) +# portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +# portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +# portfolio['sheating-energy-eff'] = None +# portfolio['sheating-env-eff'] = None +# portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) +# +# print(portfolio["mainheat-description"].value_counts()) +# +# eg = portfolio[ +# (portfolio["mainheat-description"] == "Warm air, mains gas") +# ].sample(1) +# eg = eg.squeeze().to_dict() +# print(eg)