diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py index 2ba82e77..044cc830 100644 --- a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py +++ b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py @@ -21,6 +21,8 @@ class AirSourceHeatPumpEfficiency: def create_dataset(self): logger.info("Creating solar photo supply dataset") + + all_counts = [] for dir in tqdm(self.file_directories): filepath = dir / "certificates.csv" df = pd.read_csv(filepath, low_memory=False) @@ -44,9 +46,15 @@ class AirSourceHeatPumpEfficiency: df = df[ df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False) ] + + # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA + for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]: + df = df[~pd.isnull(df[col])] # Get the columns we're interested in df = df[ [ + "PROPERTY_TYPE", + "BUILT_FORM", "MAINHEAT_DESCRIPTION", "MAINHEAT_ENERGY_EFF", "MAINHEATCONT_DESCRIPTION", @@ -60,6 +68,8 @@ class AirSourceHeatPumpEfficiency: counts = df.groupby( [ + "PROPERTY_TYPE", + "BUILT_FORM", "MAINHEAT_DESCRIPTION", "MAINHEAT_ENERGY_EFF", "MAINHEATCONT_DESCRIPTION", @@ -71,8 +81,34 @@ class AirSourceHeatPumpEfficiency: ] ).size().reset_index(name="count") - # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA - for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]: - df = df[~pd.isnull(df[col])] - # Take newest LODGEMENT_DATE per UPRN - df = df.sort_values(by="LODGEMENT_DATE", ascending=False).drop_duplicates(subset=["UPRN"]) + all_counts.append(counts) + + all_counts = pd.concat(all_counts) + + all_counts_agg = all_counts.groupby( + [ + "PROPERTY_TYPE", + "BUILT_FORM", + "MAINHEAT_DESCRIPTION", + "MAINHEAT_ENERGY_EFF", + "MAINHEATCONT_DESCRIPTION", + "MAINHEATC_ENERGY_EFF", + "MAIN_FUEL", + "HOTWATER_DESCRIPTION", + "HOT_WATER_ENERGY_EFF", + "MAINS_GAS_FLAG" + ] + )["count"].sum().reset_index() + + all_counts_agg.groupby("PROPERTY_TYPE")["count"].sum() + # In houses, 68% of the cases where we see air source heat pumps are in detached and semi-detached houses + all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "House"]["BUILT_FORM"].value_counts(normalize=True) + + all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Flat"]["BUILT_FORM"].value_counts() + + # In Bungalows, 74% of cases where we see air source heat pumps are in detached and semi-detached houses + all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Bungalow"]["BUILT_FORM"].value_counts(normalize=True) + + # TODO: Research options for mid and end-terrace houses + # TODO: Research the options for flats - we see them appear in flats, but practically speaking, how does the + # install process work? diff --git a/etl/customers/livewest/route_march.py b/etl/customers/livewest/route_march.py new file mode 100644 index 00000000..713ee56a --- /dev/null +++ b/etl/customers/livewest/route_march.py @@ -0,0 +1,135 @@ +import os + +import pandas as pd +from tqdm import tqdm + +from dotenv import load_dotenv +from utils.s3 import read_excel_from_s3 +from backend.SearchEpc import SearchEpc +from epc_api.client import EpcClient +from utils.s3 import save_csv_to_s3 + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + + +def route_march_may_2024(): + """ + This code pulls supplementary data for a route march that is expected to happen in May 2024. This code + was authored on the 30th April 2024. + """ + + asset_list = read_excel_from_s3( + bucket_name="retrofit-datalake-dev", + file_key="customers/Livewest/Livewest proposed route march Apr-May 2024.xlsx", + header_row=1 + ) + asset_list = pd.read_excel("/Users/khalimconn-kowlessar/Downloads/Livewest proposed route march Apr-May 2024.xlsx") + + epc_data = [] + for _, unit in tqdm(asset_list.iterrows(), total=len(asset_list)): + + lst = [unit["NO"], unit["ADDRESS 1"], unit["ADDRESS 2"], unit["ADDRESS 3"], unit["POSTCODE"]] + lst = [str(x).strip() for x in lst if not pd.isnull(x)] + + full_address = ", ".join(lst) + + searcher = SearchEpc( + address1=str(unit["NO"]), + postcode=unit["POSTCODE"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=None, + fast=True, + full_address=full_address + ) + # Force the skipping of estimating the EPC + searcher.ordnance_survey_client.property_type = None + searcher.ordnance_survey_client.built_form = None + + searcher.find_property(skip_os=True) + if searcher.newest_epc is None: + # We try with a different address 1 + add1 = str(unit["NO"]).lower() + add1 = ( + add1 + .replace("flat", "") + .replace("ft", "") + .replace("t", "").strip() + ) + + searcher = SearchEpc( + address1=add1, + postcode=unit["POSTCODE"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=None, + fast=True, + full_address=full_address + ) + # Force the skipping of estimating the EPC + searcher.ordnance_survey_client.property_type = None + searcher.ordnance_survey_client.built_form = None + + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None: + continue + + epc = { + "asset_list_house_no": unit["NO"], + "asset_list_address1": unit["ADDRESS 1"], + "asset_list_postcode": unit["POSTCODE"], + **searcher.newest_epc.copy() + } + + epc_data.append(epc) + + epc_df = pd.DataFrame(epc_data) + + # + + # Retrieve just the data we need + epc_df = epc_df[ + [ + "asset_list_house_no", + "asset_list_address1", + "asset_list_postcode", + "uprn", + "address", + "property-type", + "built-form", + "inspection-date", + "current-energy-rating", + "current-energy-efficiency", + "roof-description", + "walls-description", + "transaction-type" + ] + ].rename(columns={"address": "Matched EPC Address"}) + + asset_list = asset_list.merge( + epc_df, + how="left", + left_on=["NO", "ADDRESS 1", "POSTCODE"], + right_on=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"] + ) + + asset_list = asset_list.drop_duplicates(subset=["NO", "ADDRESS 1", "POSTCODE"]) + asset_list = asset_list.drop(columns=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"]) + + # Rename the columns + asset_list = asset_list.rename(columns={ + "property-type": "Property Type", + "built-form": "Archetype", + "inspection-date": "Last EPC Inspection Date", + "current-energy-rating": "Last survey EPC Rating", + "current-energy-efficiency": "Last survey SAP Score", + "roof-description": "Roof Construction", + "walls-description": "Wall Construction", + "transaction-type": "Last EPC Reason" + }) + + # Store as an excel + filename = "Livewest EPC data.xlsx" + asset_list.to_excel(filename, index=False) diff --git a/etl/customers/places_for_people/route_march.py b/etl/customers/places_for_people/route_march.py new file mode 100644 index 00000000..c38c71d3 --- /dev/null +++ b/etl/customers/places_for_people/route_march.py @@ -0,0 +1,137 @@ +import os + +import pandas as pd +from tqdm import tqdm + +from dotenv import load_dotenv +from utils.s3 import read_excel_from_s3 +from backend.SearchEpc import SearchEpc +from epc_api.client import EpcClient +from utils.s3 import save_csv_to_s3 + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + + +def app(): + """ + This app is satisying an adhoc request to retrieve EPC data for properties owned by Guiness, to help plan the + route march + + These properties were provided to us by Ecosurv + :return: + """ + asset_list = read_excel_from_s3( + bucket_name="retrofit-datalake-dev", + file_key="customers/Places For People/PFP ROUTE MARCH PHASE 1.xlsx", + header_row=1 + ) + + epc_data = [] + for _, pfp_property in tqdm(asset_list.iterrows(), total=len(asset_list)): + + lst = [ + pfp_property["ADDRESS"], + pfp_property["ADDRESS.1"], + pfp_property["ADDRESS.2"], + pfp_property["POSTCODE"] + ] + lst = [str(x).strip() for x in lst if not pd.isnull(x)] + + full_address = ", ".join(lst) + + searcher = SearchEpc( + address1=str(pfp_property["ADDRESS"]), + postcode=pfp_property["POSTCODE"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=None, + fast=True, + full_address=full_address + ) + # Force the skipping of estimating the EPC + searcher.ordnance_survey_client.property_type = None + searcher.ordnance_survey_client.built_form = None + + searcher.find_property(skip_os=True) + if searcher.newest_epc is None: + # We try with a different address 1 + add1 = str(pfp_property["ADDRESS"]).lower() + add1 = add1.replace("ft", "").replace("t", "").strip() + + searcher = SearchEpc( + address1=add1, + postcode=pfp_property["POSTCODE"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=None, + fast=True, + full_address=full_address + ) + # Force the skipping of estimating the EPC + searcher.ordnance_survey_client.property_type = None + searcher.ordnance_survey_client.built_form = None + + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None: + continue + + epc = { + "asset_list_address": pfp_property["ADDRESS"], + "asset_list_address1": pfp_property["ADDRESS.1"], + "asset_list_postcode": pfp_property["POSTCODE"], + **searcher.newest_epc.copy() + } + + epc_data.append(epc) + + epc_df = pd.DataFrame(epc_data) + + # 702 + + # Retrieve just the data we need + epc_df = epc_df[ + [ + "asset_list_address", + "asset_list_address1", + "asset_list_postcode", + "uprn", + "address", + "property-type", + "built-form", + "inspection-date", + "current-energy-rating", + "current-energy-efficiency", + "roof-description", + "walls-description", + "transaction-type" + ] + ].rename(columns={"address": "Matched EPC Address"}) + + asset_list = asset_list.merge( + epc_df, + how="left", + left_on=["ADDRESS", "ADDRESS.1", "POSTCODE"], + right_on=["asset_list_address", "asset_list_address1", "asset_list_postcode"] + ) + + # De-dupe on the address and postcode, since 137 Badger Avenue was duplicated + asset_list = asset_list.drop_duplicates(subset=["ADDRESS", "ADDRESS.1", "POSTCODE"]) + asset_list = asset_list.drop(columns=["asset_list_address", "asset_list_address1", "asset_list_postcode"]) + + # Rename the columns + asset_list = asset_list.rename(columns={ + "property-type": "Property Type", + "built-form": "Archetype", + "inspection-date": "Last EPC Inspection Date", + "current-energy-rating": "Last survey EPC Rating", + "current-energy-efficiency": "Last survey SAP Score", + "roof-description": "Roof Construction", + "walls-description": "Wall Construction", + "transaction-type": "Last EPC Reason" + }) + + # Store as an excel + filename = "Places For People EPC data.xlsx" + asset_list.to_excel(filename, index=False) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index d7a8ad2f..113bb6f8 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -37,6 +37,24 @@ MCS_SOLAR_PV_COST_DATA = { "average_cost_per_kwh-Northern Ireland": 2126.09, } +# This data is based on the MCS database +MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = { + "Outer London": None, + "Inner London": None, + "South East England": None, + "South West England": None, + "East of England": None, + "East Midlands": None, + "West Midlands": None, + "North East England": None, + "North West England": None, + "Yorkshire and the Humber": None, + "Wales": None, + "Scotland": None, + "Northern Ireland": None, +} +BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500 + # This is based on quotes from installers BATTERY_COST = 3500 @@ -1240,3 +1258,14 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days, } + + def air_source_heat_pump(self): + """ + Based on the region and type of property, this function will produce a cost estimation for an air source heat + pump. This cost will include the boiler upgrade scheme grant + + :return: + """ + + regional_cost = MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA[self.region] + pass diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index d24ad811..76da6c37 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -35,6 +35,9 @@ class HeatingControlRecommender: return + if heating_description in ["Air source heat pump, radiators, electric"]: + self.recommend_time_temperature_zone_controls() + def recommend_room_heaters_electric_controls(self): """ If the home has Room heaters, electric, we start by identifying potential heating controls that could diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 8988d2a6..b197d817 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,6 +1,4 @@ -import pandas as pd - -from recommendations.Costs import Costs +from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE from recommendations.recommendation_utils import check_simulation_difference, override_costs from backend.Property import Property from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes @@ -18,7 +16,14 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - def recommend(self, phase=0): + def recommend(self, has_cavity_and_loft_recommendations, phase=0): + """ + Produces heating recommendations + :param has_cavity_and_loft_recommendations: boolean indicating if we have produced a cavity or loft insulation + recommendation. If there are cavity or loft recommendations, the property would need to complete those measures + before being able to get the boiler upgrade scheme benefits. The messaging in the front end would be to + :param phase: indicates the phase of the retrofit programme + """ # TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace # the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this @@ -81,8 +86,120 @@ class HeatingRecommender: phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters ) + # We recommend air source heat pumps + # Heat pumps are suitable for all property types: + # https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/ + # Just seems least probable for flats, so we'll allow houses and bungalows + # In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions + # and either allow or prevent the recommendation of an air source heat pump + + suitable_property_types = self.property.data["property-type"] in ["House", "Bungalow"] + has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"] + + if suitable_property_types and not has_air_source_heat_pump: + self.recommend_air_source_heat_pump( + phase=phase, has_cavity_and_loft_recommendations=has_cavity_and_loft_recommendations + ) + return + def recommend_air_source_heat_pump(self, phase, has_cavity_and_loft_recommendations): + """ + This method will implement the recommendation for an air source heat pump + This is ultimately an overhaul to the heating system and so is recommended as an alternative to other + heating system recommendations + :return: + """ + + controls_recommender = HeatingControlRecommender(self.property) + controls_recommender.recommend(heating_description="Air source heat pump, radiators, electric") + + ashp_costs = self.costs.air_source_heat_pump() + # We add the costs of the heating controls, onto each key in the costs dictionary + if controls_recommender.recommendation: + for key in ashp_costs: + ashp_costs[key] += controls_recommender.recommendation[0][key] + + already_installed = "air_source_heat_pump" in self.property.already_installed + if already_installed: + ashp_costs = override_costs(ashp_costs) + description = "The property already has an air source heat pump, no further action needed." + else: + if controls_recommender.recommendation: + description = ("Install an air source heat pump, and upgrade heating controls to Smart Thermostats, " + "room sensors and smart radiator valves (time & temperature zone control) ") + else: + description = "Install an air source heat pump." + + # If the property does not have existing cavity and loft insulation, we include a note that the cost + # includes the boiler upgrade scheme and that the cavity and loft need to be treated, to ensure access + # to the funding + if has_cavity_and_loft_recommendations: + description = description + (f" The cost of works includes the £" + f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant. " + f"You must ensure that the property has an insulated cavity and " + f"270mm+ loft insulation to qualify for the grant") + else: + description = description + (f" The cost of works includes the £" + f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant") + + simulation_config = { + "mainheat_energy_eff_ending": "Good", + "hot_water_energy_eff_ending": "Good" + } + # Installation of a boiler improves the hot water system so we need to reflect this in + # the outcome of the recommendation + heating_ending_config = MainHeatAttributes("Air source heat pump, radiators, electric").process() + hotwater_ending_config = HotWaterAttributes("From main system").process() + + # If the property does not currently have electric main fuel, we'll simulate the change + fuel_ending_config = {} + if self.property.main_fuel["fuel_type"] != "electricity": + fuel_ending_config = MainFuelAttributes("electricity (not community)").process() + + # Check the simulation differences + heating_simulation_config = check_simulation_difference( + new_config=heating_ending_config, old_config=self.property.main_heating + ) + hotwater_simulation_config = check_simulation_difference( + new_config=hotwater_ending_config, old_config=self.property.hotwater + ) + fuel_simulation_config = check_simulation_difference( + new_config=fuel_ending_config, old_config=self.property.main_fuel + ) + + simulation_config = { + **simulation_config, + **heating_simulation_config, + **hotwater_simulation_config, + **fuel_simulation_config, + } + + if controls_recommender.recommendation: + # We should have just the single recommendation for heat controls, which is time + # and temperature zone controls + simulation_config = { + **simulation_config, + **controls_recommender.recommendation[0]["simulation_config"] + } + + ashp_recommendation = { + "phase": phase, + "parts": [ + # TODO + ], + "type": "heating", + "description": description, + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + "already_installed": already_installed, + "simulation_config": simulation_config, + **ashp_costs + } + + self.heating_recommendations.append(ashp_recommendation) + @staticmethod def check_simulation_difference(old_config, new_config): """ @@ -146,7 +263,7 @@ class HeatingRecommender: recommendation_description = f"{description} and {controls_description}" - already_installed = "cavity_wall_insulation" in self.property.already_installed + already_installed = "heating_controls" in self.property.already_installed if already_installed: total_costs = override_costs(total_costs) recommendation_description = "Heating system has already been upgraded, no further action needed." diff --git a/recommendations/tests/test_air_source_heat_pump.py b/recommendations/tests/test_air_source_heat_pump.py new file mode 100644 index 00000000..d80afc6e --- /dev/null +++ b/recommendations/tests/test_air_source_heat_pump.py @@ -0,0 +1,77 @@ +from backend.Property import Property +from recommendations.HeatingRecommender import HeatingRecommender +from etl.epc.Record import EPCRecord + + +class TestAirSourceHeatPump: + + def test_eligible(self): + # This tests a house, which will be suitable for an air source heat pump + epc_record = EPCRecord() + epc_record.prepared_epc = { + "county": "Broxbourne", + "mainheat-energy-eff": "Good", + "hot-water-energy-eff": "Good", + "mainheatc-energy-eff": "Good", + "number-heated-rooms": 5, + "property-type": "House", + "built-form": "Semi-Detached" + } + + property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) + property_instance.main_heating = { + 'original_description': 'Boiler and radiators, mains gas', + "clean_description": "Boiler and radiators, mains gas", + 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, + 'has_electric_underfloor_heating': False, + 'has_electric_ceiling_heating': False, 'has_community_scheme': False, + 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, + 'has_water_source_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, + 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, + 'has_lpg': False, 'has_assumed': False, + 'has_electricaire': False, 'has_assumed_for_most_rooms': False, + 'has_underfloor_heating': False, + "has_electric_heat_pumps": False, + "has_micro-cogeneration": False + } + property_instance.main_fuel = { + 'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas', + 'tariff_type': None, + 'is_community': False, 'no_individual_heating_or_community_network': False, + 'complex_fuel_type': None + } + property_instance.hotwater = { + 'original_description': 'From main system', + 'clean_description': 'From main system', + 'heater_type': None, + 'system_type': 'from main system', + 'thermostat_characteristics': None, 'heating_scope': None, + 'energy_recovery': None, 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, + 'no_system_present': None, + 'assumed': False, "appliance": None + } + property_instance.main_heating_controls = { + 'original_description': 'Programmer, room thermostat and TRVs', + 'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer', + 'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, + 'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None + + } + + recommender = HeatingRecommender(property_instance=property_instance) + + assert not recommender.heating_recommendations + + recommender.recommend(phase=0) + + assert recommender.recommendation is None