working through the air source heat pump recommendations, added route march code for livewest

This commit is contained in:
Khalim Conn-Kowlessar 2024-04-30 17:41:33 +01:00
parent 03ca16bfc5
commit 155a8c568c
9 changed files with 546 additions and 12 deletions

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyNamespacePackagesService">

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>

View file

@ -21,6 +21,8 @@ class AirSourceHeatPumpEfficiency:
def create_dataset(self):
logger.info("Creating solar photo supply dataset")
all_counts = []
for dir in tqdm(self.file_directories):
filepath = dir / "certificates.csv"
df = pd.read_csv(filepath, low_memory=False)
@ -44,9 +46,15 @@ class AirSourceHeatPumpEfficiency:
df = df[
df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False)
]
# Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
df = df[~pd.isnull(df[col])]
# Get the columns we're interested in
df = df[
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
@ -60,6 +68,8 @@ class AirSourceHeatPumpEfficiency:
counts = df.groupby(
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
@ -71,8 +81,34 @@ class AirSourceHeatPumpEfficiency:
]
).size().reset_index(name="count")
# Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
df = df[~pd.isnull(df[col])]
# Take newest LODGEMENT_DATE per UPRN
df = df.sort_values(by="LODGEMENT_DATE", ascending=False).drop_duplicates(subset=["UPRN"])
all_counts.append(counts)
all_counts = pd.concat(all_counts)
all_counts_agg = all_counts.groupby(
[
"PROPERTY_TYPE",
"BUILT_FORM",
"MAINHEAT_DESCRIPTION",
"MAINHEAT_ENERGY_EFF",
"MAINHEATCONT_DESCRIPTION",
"MAINHEATC_ENERGY_EFF",
"MAIN_FUEL",
"HOTWATER_DESCRIPTION",
"HOT_WATER_ENERGY_EFF",
"MAINS_GAS_FLAG"
]
)["count"].sum().reset_index()
all_counts_agg.groupby("PROPERTY_TYPE")["count"].sum()
# In houses, 68% of the cases where we see air source heat pumps are in detached and semi-detached houses
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "House"]["BUILT_FORM"].value_counts(normalize=True)
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Flat"]["BUILT_FORM"].value_counts()
# In Bungalows, 74% of cases where we see air source heat pumps are in detached and semi-detached houses
all_counts_agg[all_counts_agg["PROPERTY_TYPE"] == "Bungalow"]["BUILT_FORM"].value_counts(normalize=True)
# TODO: Research options for mid and end-terrace houses
# TODO: Research the options for flats - we see them appear in flats, but practically speaking, how does the
# install process work?

View file

@ -0,0 +1,135 @@
import os
import pandas as pd
from tqdm import tqdm
from dotenv import load_dotenv
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from epc_api.client import EpcClient
from utils.s3 import save_csv_to_s3
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
def route_march_may_2024():
"""
This code pulls supplementary data for a route march that is expected to happen in May 2024. This code
was authored on the 30th April 2024.
"""
asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/Livewest/Livewest proposed route march Apr-May 2024.xlsx",
header_row=1
)
asset_list = pd.read_excel("/Users/khalimconn-kowlessar/Downloads/Livewest proposed route march Apr-May 2024.xlsx")
epc_data = []
for _, unit in tqdm(asset_list.iterrows(), total=len(asset_list)):
lst = [unit["NO"], unit["ADDRESS 1"], unit["ADDRESS 2"], unit["ADDRESS 3"], unit["POSTCODE"]]
lst = [str(x).strip() for x in lst if not pd.isnull(x)]
full_address = ", ".join(lst)
searcher = SearchEpc(
address1=str(unit["NO"]),
postcode=unit["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
# We try with a different address 1
add1 = str(unit["NO"]).lower()
add1 = (
add1
.replace("flat", "")
.replace("ft", "")
.replace("t", "").strip()
)
searcher = SearchEpc(
address1=add1,
postcode=unit["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
continue
epc = {
"asset_list_house_no": unit["NO"],
"asset_list_address1": unit["ADDRESS 1"],
"asset_list_postcode": unit["POSTCODE"],
**searcher.newest_epc.copy()
}
epc_data.append(epc)
epc_df = pd.DataFrame(epc_data)
#
# Retrieve just the data we need
epc_df = epc_df[
[
"asset_list_house_no",
"asset_list_address1",
"asset_list_postcode",
"uprn",
"address",
"property-type",
"built-form",
"inspection-date",
"current-energy-rating",
"current-energy-efficiency",
"roof-description",
"walls-description",
"transaction-type"
]
].rename(columns={"address": "Matched EPC Address"})
asset_list = asset_list.merge(
epc_df,
how="left",
left_on=["NO", "ADDRESS 1", "POSTCODE"],
right_on=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"]
)
asset_list = asset_list.drop_duplicates(subset=["NO", "ADDRESS 1", "POSTCODE"])
asset_list = asset_list.drop(columns=["asset_list_house_no", "asset_list_address1", "asset_list_postcode"])
# Rename the columns
asset_list = asset_list.rename(columns={
"property-type": "Property Type",
"built-form": "Archetype",
"inspection-date": "Last EPC Inspection Date",
"current-energy-rating": "Last survey EPC Rating",
"current-energy-efficiency": "Last survey SAP Score",
"roof-description": "Roof Construction",
"walls-description": "Wall Construction",
"transaction-type": "Last EPC Reason"
})
# Store as an excel
filename = "Livewest EPC data.xlsx"
asset_list.to_excel(filename, index=False)

View file

@ -0,0 +1,137 @@
import os
import pandas as pd
from tqdm import tqdm
from dotenv import load_dotenv
from utils.s3 import read_excel_from_s3
from backend.SearchEpc import SearchEpc
from epc_api.client import EpcClient
from utils.s3 import save_csv_to_s3
load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
def app():
"""
This app is satisying an adhoc request to retrieve EPC data for properties owned by Guiness, to help plan the
route march
These properties were provided to us by Ecosurv
:return:
"""
asset_list = read_excel_from_s3(
bucket_name="retrofit-datalake-dev",
file_key="customers/Places For People/PFP ROUTE MARCH PHASE 1.xlsx",
header_row=1
)
epc_data = []
for _, pfp_property in tqdm(asset_list.iterrows(), total=len(asset_list)):
lst = [
pfp_property["ADDRESS"],
pfp_property["ADDRESS.1"],
pfp_property["ADDRESS.2"],
pfp_property["POSTCODE"]
]
lst = [str(x).strip() for x in lst if not pd.isnull(x)]
full_address = ", ".join(lst)
searcher = SearchEpc(
address1=str(pfp_property["ADDRESS"]),
postcode=pfp_property["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
# We try with a different address 1
add1 = str(pfp_property["ADDRESS"]).lower()
add1 = add1.replace("ft", "").replace("t", "").strip()
searcher = SearchEpc(
address1=add1,
postcode=pfp_property["POSTCODE"],
auth_token=EPC_AUTH_TOKEN,
os_api_key="",
property_type=None,
fast=True,
full_address=full_address
)
# Force the skipping of estimating the EPC
searcher.ordnance_survey_client.property_type = None
searcher.ordnance_survey_client.built_form = None
searcher.find_property(skip_os=True)
if searcher.newest_epc is None:
continue
epc = {
"asset_list_address": pfp_property["ADDRESS"],
"asset_list_address1": pfp_property["ADDRESS.1"],
"asset_list_postcode": pfp_property["POSTCODE"],
**searcher.newest_epc.copy()
}
epc_data.append(epc)
epc_df = pd.DataFrame(epc_data)
# 702
# Retrieve just the data we need
epc_df = epc_df[
[
"asset_list_address",
"asset_list_address1",
"asset_list_postcode",
"uprn",
"address",
"property-type",
"built-form",
"inspection-date",
"current-energy-rating",
"current-energy-efficiency",
"roof-description",
"walls-description",
"transaction-type"
]
].rename(columns={"address": "Matched EPC Address"})
asset_list = asset_list.merge(
epc_df,
how="left",
left_on=["ADDRESS", "ADDRESS.1", "POSTCODE"],
right_on=["asset_list_address", "asset_list_address1", "asset_list_postcode"]
)
# De-dupe on the address and postcode, since 137 Badger Avenue was duplicated
asset_list = asset_list.drop_duplicates(subset=["ADDRESS", "ADDRESS.1", "POSTCODE"])
asset_list = asset_list.drop(columns=["asset_list_address", "asset_list_address1", "asset_list_postcode"])
# Rename the columns
asset_list = asset_list.rename(columns={
"property-type": "Property Type",
"built-form": "Archetype",
"inspection-date": "Last EPC Inspection Date",
"current-energy-rating": "Last survey EPC Rating",
"current-energy-efficiency": "Last survey SAP Score",
"roof-description": "Roof Construction",
"walls-description": "Wall Construction",
"transaction-type": "Last EPC Reason"
})
# Store as an excel
filename = "Places For People EPC data.xlsx"
asset_list.to_excel(filename, index=False)

View file

@ -37,6 +37,24 @@ MCS_SOLAR_PV_COST_DATA = {
"average_cost_per_kwh-Northern Ireland": 2126.09,
}
# This data is based on the MCS database
MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = {
"Outer London": None,
"Inner London": None,
"South East England": None,
"South West England": None,
"East of England": None,
"East Midlands": None,
"West Midlands": None,
"North East England": None,
"North West England": None,
"Yorkshire and the Humber": None,
"Wales": None,
"Scotland": None,
"Northern Ireland": None,
}
BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500
# This is based on quotes from installers
BATTERY_COST = 3500
@ -1240,3 +1258,14 @@ class Costs:
"labour_hours": labour_hours,
"labour_days": labour_days,
}
def air_source_heat_pump(self):
"""
Based on the region and type of property, this function will produce a cost estimation for an air source heat
pump. This cost will include the boiler upgrade scheme grant
:return:
"""
regional_cost = MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA[self.region]
pass

View file

@ -35,6 +35,9 @@ class HeatingControlRecommender:
return
if heating_description in ["Air source heat pump, radiators, electric"]:
self.recommend_time_temperature_zone_controls()
def recommend_room_heaters_electric_controls(self):
"""
If the home has Room heaters, electric, we start by identifying potential heating controls that could

View file

@ -1,6 +1,4 @@
import pandas as pd
from recommendations.Costs import Costs
from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE
from recommendations.recommendation_utils import check_simulation_difference, override_costs
from backend.Property import Property
from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
@ -18,7 +16,14 @@ class HeatingRecommender:
self.heating_recommendations = []
self.heating_control_recommendations = []
def recommend(self, phase=0):
def recommend(self, has_cavity_and_loft_recommendations, phase=0):
"""
Produces heating recommendations
:param has_cavity_and_loft_recommendations: boolean indicating if we have produced a cavity or loft insulation
recommendation. If there are cavity or loft recommendations, the property would need to complete those measures
before being able to get the boiler upgrade scheme benefits. The messaging in the front end would be to
:param phase: indicates the phase of the retrofit programme
"""
# TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace
# the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this
@ -81,8 +86,120 @@ class HeatingRecommender:
phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters
)
# We recommend air source heat pumps
# Heat pumps are suitable for all property types:
# https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/
# Just seems least probable for flats, so we'll allow houses and bungalows
# In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions
# and either allow or prevent the recommendation of an air source heat pump
suitable_property_types = self.property.data["property-type"] in ["House", "Bungalow"]
has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"]
if suitable_property_types and not has_air_source_heat_pump:
self.recommend_air_source_heat_pump(
phase=phase, has_cavity_and_loft_recommendations=has_cavity_and_loft_recommendations
)
return
def recommend_air_source_heat_pump(self, phase, has_cavity_and_loft_recommendations):
"""
This method will implement the recommendation for an air source heat pump
This is ultimately an overhaul to the heating system and so is recommended as an alternative to other
heating system recommendations
:return:
"""
controls_recommender = HeatingControlRecommender(self.property)
controls_recommender.recommend(heating_description="Air source heat pump, radiators, electric")
ashp_costs = self.costs.air_source_heat_pump()
# We add the costs of the heating controls, onto each key in the costs dictionary
if controls_recommender.recommendation:
for key in ashp_costs:
ashp_costs[key] += controls_recommender.recommendation[0][key]
already_installed = "air_source_heat_pump" in self.property.already_installed
if already_installed:
ashp_costs = override_costs(ashp_costs)
description = "The property already has an air source heat pump, no further action needed."
else:
if controls_recommender.recommendation:
description = ("Install an air source heat pump, and upgrade heating controls to Smart Thermostats, "
"room sensors and smart radiator valves (time & temperature zone control) ")
else:
description = "Install an air source heat pump."
# If the property does not have existing cavity and loft insulation, we include a note that the cost
# includes the boiler upgrade scheme and that the cavity and loft need to be treated, to ensure access
# to the funding
if has_cavity_and_loft_recommendations:
description = description + (f" The cost of works includes the £"
f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant. "
f"You must ensure that the property has an insulated cavity and "
f"270mm+ loft insulation to qualify for the grant")
else:
description = description + (f" The cost of works includes the £"
f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant")
simulation_config = {
"mainheat_energy_eff_ending": "Good",
"hot_water_energy_eff_ending": "Good"
}
# Installation of a boiler improves the hot water system so we need to reflect this in
# the outcome of the recommendation
heating_ending_config = MainHeatAttributes("Air source heat pump, radiators, electric").process()
hotwater_ending_config = HotWaterAttributes("From main system").process()
# If the property does not currently have electric main fuel, we'll simulate the change
fuel_ending_config = {}
if self.property.main_fuel["fuel_type"] != "electricity":
fuel_ending_config = MainFuelAttributes("electricity (not community)").process()
# Check the simulation differences
heating_simulation_config = check_simulation_difference(
new_config=heating_ending_config, old_config=self.property.main_heating
)
hotwater_simulation_config = check_simulation_difference(
new_config=hotwater_ending_config, old_config=self.property.hotwater
)
fuel_simulation_config = check_simulation_difference(
new_config=fuel_ending_config, old_config=self.property.main_fuel
)
simulation_config = {
**simulation_config,
**heating_simulation_config,
**hotwater_simulation_config,
**fuel_simulation_config,
}
if controls_recommender.recommendation:
# We should have just the single recommendation for heat controls, which is time
# and temperature zone controls
simulation_config = {
**simulation_config,
**controls_recommender.recommendation[0]["simulation_config"]
}
ashp_recommendation = {
"phase": phase,
"parts": [
# TODO
],
"type": "heating",
"description": description,
"starting_u_value": None,
"new_u_value": None,
"sap_points": None,
"already_installed": already_installed,
"simulation_config": simulation_config,
**ashp_costs
}
self.heating_recommendations.append(ashp_recommendation)
@staticmethod
def check_simulation_difference(old_config, new_config):
"""
@ -146,7 +263,7 @@ class HeatingRecommender:
recommendation_description = f"{description} and {controls_description}"
already_installed = "cavity_wall_insulation" in self.property.already_installed
already_installed = "heating_controls" in self.property.already_installed
if already_installed:
total_costs = override_costs(total_costs)
recommendation_description = "Heating system has already been upgraded, no further action needed."

View file

@ -0,0 +1,77 @@
from backend.Property import Property
from recommendations.HeatingRecommender import HeatingRecommender
from etl.epc.Record import EPCRecord
class TestAirSourceHeatPump:
def test_eligible(self):
# This tests a house, which will be suitable for an air source heat pump
epc_record = EPCRecord()
epc_record.prepared_epc = {
"county": "Broxbourne",
"mainheat-energy-eff": "Good",
"hot-water-energy-eff": "Good",
"mainheatc-energy-eff": "Good",
"number-heated-rooms": 5,
"property-type": "House",
"built-form": "Semi-Detached"
}
property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
property_instance.main_heating = {
'original_description': 'Boiler and radiators, mains gas',
"clean_description": "Boiler and radiators, mains gas",
'has_radiators': True,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
'has_air_source_heat_pump': False,
'has_room_heaters': False, 'has_electric_storage_heaters': False,
'has_warm_air': False,
'has_electric_underfloor_heating': False,
'has_electric_ceiling_heating': False, 'has_community_scheme': False,
'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False,
'has_water_source_heat_pump': False, 'has_electric': False,
'has_mains_gas': True, 'has_wood_logs': False,
'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
'has_anthracite': False,
'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False,
'has_lpg': False, 'has_assumed': False,
'has_electricaire': False, 'has_assumed_for_most_rooms': False,
'has_underfloor_heating': False,
"has_electric_heat_pumps": False,
"has_micro-cogeneration": False
}
property_instance.main_fuel = {
'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas',
'tariff_type': None,
'is_community': False, 'no_individual_heating_or_community_network': False,
'complex_fuel_type': None
}
property_instance.hotwater = {
'original_description': 'From main system',
'clean_description': 'From main system',
'heater_type': None,
'system_type': 'from main system',
'thermostat_characteristics': None, 'heating_scope': None,
'energy_recovery': None, 'tariff_type': None,
'extra_features': None, 'chp_systems': None, 'distribution_system': None,
'no_system_present': None,
'assumed': False, "appliance": None
}
property_instance.main_heating_controls = {
'original_description': 'Programmer, room thermostat and TRVs',
'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer',
'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None
}
recommender = HeatingRecommender(property_instance=property_instance)
assert not recommender.heating_recommendations
recommender.recommend(phase=0)
assert recommender.recommendation is None