From 325748524817708da77c30c9f1155470d7020e1f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 24 Jul 2024 18:46:39 +0100 Subject: [PATCH] xml extraction wip --- etl/xml_survey_extraction/XmlParser.py | 540 ++++++++++++ etl/xml_survey_extraction/app.py | 43 +- etl/xml_survey_extraction/pcdb.py | 1129 ++++++++++++++++++++++++ utils/s3.py | 83 ++ 4 files changed, 1794 insertions(+), 1 deletion(-) create mode 100644 etl/xml_survey_extraction/XmlParser.py create mode 100644 etl/xml_survey_extraction/pcdb.py diff --git a/etl/xml_survey_extraction/XmlParser.py b/etl/xml_survey_extraction/XmlParser.py new file mode 100644 index 00000000..de7e35f8 --- /dev/null +++ b/etl/xml_survey_extraction/XmlParser.py @@ -0,0 +1,540 @@ +import re +import usaddress +from xml.dom.minidom import parseString +from backend.app.utils import sap_to_epc +from etl.xml_survey_extraction.pcdb import heating_data + +PROPERTY_TYPE_LOOKUP = { + "0": "House", + "House": "House", +} + + +def get_house_number(address: str) -> str | None: + """ + This method will use the usaddress library to parse an address and extract the house number + :return: + """ + + parsed = usaddress.parse(address) + parsed_house_number = [x for x in parsed if (x[1] == "AddressNumber")] + parsed_house_number = parsed_house_number[0][0] if parsed_house_number else None + + if parsed_house_number is None: + # Because usaddress isn't optimal for parsing addresses with some prefixes such as 'Flat', + # we also add a custom approach + + # Pattern to look for 'Flat' or 'Apartment' followed by a number, or just a number at the beginning + pattern = r'(?i)(?:flat|apartment)\s*(\d+)|^\s*(\d+)' + + match = re.search(pattern, address) + + if match: + # Return the first non-None group found + return next(g for g in match.groups() if g is not None) + else: + return None + + # Remove training commas + parsed_house_number = parsed_house_number.replace(",", "") + + return parsed_house_number + + +class XmlParser: + uprn = None + property_type = None + current_energy_efficiency = None + current_energy_rating = None + + # heating/emissions information + space_heating_kwh = None + water_heating_kwh = None + co2_emissions_current = None + heating_cost_current = None + hot_water_cost_current = None + lighting_cost_current = None + energy_consumption_current = None + heating_system = None + heating_controls = None + + # Assessor details + surveyor_name = None + + # Addresses + address1 = None + address2 = None + address3 = None + posttown = None + postcode = None + address = None + + # Dates + survey_date = None + + # Building Fabric + # Walls + walls_description = None + walls_classification = None + walls_energy_rating = None + # Roof + roof_description = None + roof_energy_rating = None + is_loft = None + # Floor + floor_description = None + floor_energy_rating = None + # Windows + windows_description = None + windows_energy_rating = None + # main heating + main_heating_description = None + main_heating_energy_rating = None + # Heating controls + main_heating_controls_description = None + main_heating_controls_energy_rating = None + # Hot water + hot_water_description = None + hot_water_energy_rating = None + # Lighting + lighting_description = None + lighting_energy_rating = None + # Second Heating + second_heating_description = None + second_heating_energy_rating = None + + number_of_doors = None + number_of_insulated_doors = None + photo_supply = None + + # Property dimensions + number_of_floors = None + perimeter = None + heat_loss_perimeter = None + party_wall_length = None + total_floor_area = None + ground_floor_area = None + is_there_party_wall = None + floor_height = None + insulation_wall_area = None + + rrn = None + + database_data = None + + # We assume that the insulation wall area is 85% of the total wall area, as a standard estimate + INSULATION_WALL_AREA_FACTOR = 0.85 + + # The value of the URPN tells us about the file type that we're parsing + UPRN_FILETYPE_MAP = { + 0: "EPR", + -1: "RDSAP_EPR" + } + + RATINGS_MAP = { + "0": "N/A", + "1": "Very Poor", + "2": "Poor", + "3": "Average", + "4": "Good", + "5": "Very Good" + } + + def __init__(self, file, filekey, uprn=None): + file.seek(0) # Ensure the file pointer is at the beginning + xml_string = file.read().decode('utf-8') + self.xml = parseString(xml_string) + self.filekey = filekey + + # The xml parser is use to parse the EPC and EPR xmls and different file types will contain different + # information + # In order to identify the file type, we can look for the presence of the 'UPRN' tag + # If the UPRN tag is present, we can assume that the file is an EPC + # If the UPRN tag is not present, we can assume that the file is an EPR + self.get_uprn() + + self.file_type = self.UPRN_FILETYPE_MAP.get(self.uprn, "EPC") + + @staticmethod + def get_node(node): + """ + Utility function to get the node value from the xml, where data might be optional + :return: + """ + + node_first_child = node.firstChild + if node_first_child is None: + return None + + return node_first_child.nodeValue + + def run(self): + if self.file_type == "RDSAP_EPR": + # This file type contains just limited information compared to a regular EPR/EPC, and so we just exit + # unless we learn something else that determines that we need information from this file + return + self.get_property_type() + self.get_sap() + self.get_property_address() + self.get_dates() + self.get_assessor_details() + + self.get_heating_and_emissions_data() + self.get_detailed_heating_specs() + + # Building fabric + self.get_walls() + self.get_roof() + self.get_floor() + self.get_windows() + self.get_heating() + self.get_hot_water() + self.get_lighting() + self.get_doors() + self.get_photo_supply() + + # Property dimensions + self.get_property_dimensions() + + def get_uprn(self, uprn): + + if uprn is not None: + self.uprn = uprn + return + + uprn_tag = self.xml.getElementsByTagName('UPRN')[0].firstChild + if uprn_tag is None: + self.uprn = -1 + return + + self.uprn = uprn_tag.nodeValue + # If all of the characters in the UPRN are 0, then there is not set UPRN + if self.uprn.count("0") == len(self.uprn): + self.uprn = 0 + else: + self.uprn = self.uprn.lower().split("uprn-")[1] + + def get_property_type(self): + if not self.xml: + raise ValueError("You need to read the file first") + + property_type = self.xml.getElementsByTagName('Property-Type') + if not property_type: + property_type = self.xml.getElementsByTagName('PropertyType1') + + self.property_type = PROPERTY_TYPE_LOOKUP[property_type[0].firstChild.nodeValue] + + def get_sap(self): + sap_score = self.xml.getElementsByTagName('Energy-Rating-Current') + sap_score = int(sap_score[0].firstChild.nodeValue) + epc_rating = sap_to_epc(sap_score) + self.current_energy_efficiency = str(sap_score) + self.current_energy_rating = epc_rating + + def get_heating_and_emissions_data(self): + """ + This method will extract the following pieces of information: + 1) Space heating requirement + 2) Water heating requirement + 3) CO2 emissions + 4) Heat demand per square meter per year + 5) Bills + + :return: + """ + + self.space_heating_kwh = self.xml.getElementsByTagName( + 'Space-Heating-Existing-Dwelling' + )[0].firstChild.nodeValue + + self.water_heating_kwh = self.xml.getElementsByTagName('Water-Heating')[0].firstChild.nodeValue + + self.co2_emissions_current = self.xml.getElementsByTagName('CO2-Emissions-Current')[0].firstChild.nodeValue + self.heating_cost_current = self.xml.getElementsByTagName('Heating-Cost-Current')[0].firstChild.nodeValue + self.hot_water_cost_current = self.xml.getElementsByTagName('Hot-Water-Cost-Current')[0].firstChild.nodeValue + self.lighting_cost_current = self.xml.getElementsByTagName('Lighting-Cost-Current')[0].firstChild.nodeValue + self.energy_consumption_current = ( + self.xml.getElementsByTagName("Energy-Consumption-Current")[0].firstChild.nodeValue + ) + + def get_detailed_heating_specs(self): + """ + Given the heating data that is found in the tag, we extract the detailed about the heating + system + :return: + """ + sap_main_heating_details = ( + self.xml.getElementsByTagName('SAP-Heating')[0] + .getElementsByTagName("Main-Heating-Details")[0] + .getElementsByTagName("Main-Heating")[0] + ) + + heating_code = sap_main_heating_details.getElementsByTagName("SAP-Main-Heating-Code")[0].firstChild.nodeValue + + # Get the heating system + heating_system = heating_data[heating_data["code"] == int(heating_code)]["description"] + heating_system = heating_system.values[0] if not heating_system.empty else f"Heating code: {heating_code}" + + # Get the heating controls + heating_controls_code = ( + sap_main_heating_details.getElementsByTagName("Main-Heating-Control")[0].firstChild.nodeValue + ) + + heating_controls = heating_data[heating_data["code"] == int(heating_controls_code)]["description"] + heating_controls = ( + heating_controls.values[0] if not heating_controls.empty else f"Heating Controls code: {heating_code}" + ) + + self.heating_system = heating_system + self.heating_controls = heating_controls + + def get_walls(self): + + wall_xml_data = self.xml.getElementsByTagName('Property-Summary')[0].getElementsByTagName('Wall')[0] + + self.walls_description = ( + wall_xml_data + .getElementsByTagName("Description")[0] + .firstChild.nodeValue + ) + + self.walls_energy_rating = ( + wall_xml_data + .getElementsByTagName("Energy-Efficiency-Rating")[0] + .firstChild.nodeValue + ) + + is_cavity = "cavity wall" in self.walls_description.lower() + is_empty = "no insulation" in self.walls_description.lower() + is_partial = "partial insulation" in self.walls_description.lower() + + if not is_cavity: + self.walls_classification = "NON CAVITY" + return + + if is_empty: + self.walls_classification = "EMPTY" + return + + if is_partial: + self.walls_classification = "PARTIAL" + return + + if is_cavity and not is_empty and not is_partial: + self.walls_classification = "FULL" + return + + raise NotImplementedError("Implement me") + + def get_roof(self): + + room_xml_data = self.xml.getElementsByTagName('Property-Summary')[0].getElementsByTagName('Roof')[0] + + self.roof_description = ( + room_xml_data + .getElementsByTagName("Description")[0] + .firstChild.nodeValue + ) + + self.roof_energy_rating = ( + room_xml_data + .getElementsByTagName("Energy-Efficiency-Rating")[0] + .firstChild.nodeValue + ) + + loft_recommendation_tag = self.xml.getElementsByTagName("Impact-Of-Loft-Insulation") + description_contains_loft = "loft" in self.roof_description.lower() + + if not loft_recommendation_tag and not description_contains_loft: + self.is_loft = "No" + return + + self.is_loft = "Yes" + return + + def get_floor(self): + + floor_xml_data = self.xml.getElementsByTagName('Property-Summary')[0].getElementsByTagName('Floor')[0] + + self.floor_description = ( + floor_xml_data + .getElementsByTagName("Description")[0] + .firstChild.nodeValue + ) + + self.floor_energy_rating = ( + floor_xml_data + .getElementsByTagName("Energy-Efficiency-Rating")[0] + .firstChild.nodeValue + ) + + def get_windows(self): + + windows_xml_data = self.xml.getElementsByTagName('Property-Summary')[0].getElementsByTagName('Window')[0] + + self.windows_description = ( + windows_xml_data + .getElementsByTagName("Description")[0] + .firstChild.nodeValue + ) + + self.windows_energy_rating = ( + windows_xml_data + .getElementsByTagName("Energy-Efficiency-Rating")[0] + .firstChild.nodeValue + ) + + def get_heating(self): + """ + This function will retrieve the main heating and the main heating controls + :return: + """ + mainheating_xml_data = self.xml.getElementsByTagName('Main-Heating')[0] + + self.main_heating_description = ( + mainheating_xml_data.getElementsByTagName('Description')[0].firstChild.nodeValue + ) + + self.main_heating_energy_rating = ( + mainheating_xml_data.getElementsByTagName('Energy-Efficiency-Rating')[0].firstChild.nodeValue + ) + + mainheating_controls_xml_data = self.xml.getElementsByTagName('Main-Heating-Controls')[0] + + self.main_heating_controls_description = ( + mainheating_controls_xml_data.getElementsByTagName('Description')[0].firstChild.nodeValue + ) + + self.main_heating_controls_energy_rating = ( + mainheating_controls_xml_data.getElementsByTagName('Energy-Efficiency-Rating')[0].firstChild.nodeValue + ) + + second_heating_xml_data = self.xml.getElementsByTagName('Secondary-Heating')[0] + + self.second_heating_description = ( + second_heating_xml_data.getElementsByTagName('Description')[0].firstChild.nodeValue + ) + + self.second_heating_energy_rating = ( + second_heating_xml_data.getElementsByTagName('Energy-Efficiency-Rating')[0].firstChild.nodeValue + ) + + def get_hot_water(self): + hot_water_xml_data = self.xml.getElementsByTagName('Hot-Water')[0] + + self.hot_water_description = ( + hot_water_xml_data.getElementsByTagName('Description')[0].firstChild.nodeValue + ) + + self.hot_water_energy_rating = ( + hot_water_xml_data.getElementsByTagName('Energy-Efficiency-Rating')[0].firstChild.nodeValue + ) + + def get_lighting(self): + lighting_xml_data = self.xml.getElementsByTagName('Lighting')[0] + + self.lighting_description = ( + lighting_xml_data.getElementsByTagName('Description')[0].firstChild.nodeValue + ) + + self.lighting_energy_rating = ( + lighting_xml_data.getElementsByTagName('Energy-Efficiency-Rating')[0].firstChild.nodeValue + ) + + def get_doors(self): + + # Doors can be found in the SAP-Property-Details tag + self.number_of_doors = int( + self.xml.getElementsByTagName('SAP-Property-Details')[0] + .getElementsByTagName('Door-Count')[0] + .firstChild.nodeValue + ) + + self.number_of_insulated_doors = int( + self.xml.getElementsByTagName('SAP-Property-Details')[0] + .getElementsByTagName('Insulated-Door-Count')[0] + .firstChild.nodeValue + ) + + def get_photo_supply(self): + self.photo_supply = float( + self.xml.getElementsByTagName('Photovoltaic-Supply')[0] + .getElementsByTagName('Percent-Roof-Area')[0] + .firstChild.nodeValue + ) + + def get_assessor_details(self): + + energy_assessor_tag = self.xml.getElementsByTagName('Energy-Assessor')[0] + + self.surveyor_name = ( + energy_assessor_tag.getElementsByTagName("Name")[0].firstChild.nodeValue + ) + + def get_property_address(self): + + property_tag = self.xml.getElementsByTagName("Property")[0] + + self.address1 = self.get_node(property_tag.getElementsByTagName("Address-Line-1")[0]) + self.address2 = self.get_node(property_tag.getElementsByTagName("Address-Line-2")[0]) + self.address3 = self.get_node(property_tag.getElementsByTagName("Address-Line-3")[0]) + self.posttown = self.get_node(property_tag.getElementsByTagName("Post-Town")[0]) + self.postcode = self.get_node(property_tag.getElementsByTagName("Postcode")[0]) + self.address = ", ".join( + [x for x in [self.address1, self.address2, self.address3, self.posttown, self.postcode] if x is not None] + ) + + def get_dates(self): + self.survey_date = ( + self.xml.getElementsByTagName("Inspection-Date")[0].firstChild.nodeValue + ) + + def get_property_dimensions(self): + """ + This function will extract the relevant property dimensions including the floor area, + number of floors, perimeter, party wall length and the insulation_wall_area. + + insulation_wall_area is typically simplified down to perimeter * height * 0.85 + :return: + """ + + # Each floor has its own SAP-Floor-Dimension tag + floor_dimensions = ( + self.xml.getElementsByTagName("SAP-Floor-Dimensions")[0] + .getElementsByTagName("SAP-Floor-Dimension") + ) + + self.number_of_floors = len(floor_dimensions) + + self.heat_loss_perimeter = float( + floor_dimensions[0].getElementsByTagName("Heat-Loss-Perimeter")[0].firstChild.nodeValue + ) + + self.party_wall_length = float( + floor_dimensions[0].getElementsByTagName("Party-Wall-Length")[0].firstChild.nodeValue + ) + + party_wall_construction_tag = ( + self.xml.getElementsByTagName("Party-Wall-Construction")[0].firstChild.nodeValue.replace("\n", "").strip() + ) + + self.is_there_party_wall = ( + "Yes" if (self.party_wall_length > 0) or (party_wall_construction_tag != "") else "No" + ) + + # We pull out all of the floor areas + floor_areas = [ + float(x.getElementsByTagName("Total-Floor-Area")[0].firstChild.nodeValue) for x in floor_dimensions + ] + + self.total_floor_area = sum(floor_areas) + self.ground_floor_area = floor_areas[0] + + self.floor_height = float( + floor_dimensions[0] + .getElementsByTagName("Room-Height")[0] + .firstChild.nodeValue + ) + + self.insulation_wall_area = self.heat_loss_perimeter * self.floor_height * self.INSULATION_WALL_AREA_FACTOR + self.perimeter = self.heat_loss_perimeter + self.party_wall_length diff --git a/etl/xml_survey_extraction/app.py b/etl/xml_survey_extraction/app.py index 6f53e4e2..9bcbb168 100644 --- a/etl/xml_survey_extraction/app.py +++ b/etl/xml_survey_extraction/app.py @@ -1,3 +1,16 @@ +from utils.s3 import read_from_s3, list_files_and_subfolders_in_s3_folder, list_xmls_in_s3_folder +from utils.logger import setup_logger +from etl.xml_survey_extraction.XmlParser import XmlParser +import os +from io import BytesIO + +logger = setup_logger() + +SURVEYORS = "JAFFERSONS ENERGY CONSULTANTS" +PROJECT_CODE = "VDE001" +BUCKET = "retrofit-energy-assessments-dev" + + def main(): """ This function executes the main process, which will retrieve data from the specified locations, extract the data @@ -6,4 +19,32 @@ def main(): """ # TODO: Build solution to get this data from Onedrive and store what we need in S3 - # In s3, we have a bucket called retrofit-energy-assessments-{stage} which + # In s3, we have a bucket called retrofit-energy-assessments-{stage} which contains the data we need + # The data is stored in a folder called {surveyors}/{project_code}/{uprn} + # We'll need to get the uprn from the folder name, which we can do with EpcSearcher class + + # + energy_assessments = list_files_and_subfolders_in_s3_folder( + bucket_name=BUCKET, folder_name=f"{SURVEYORS}/{PROJECT_CODE}/" + ) + + logger.info(f"Found {len(energy_assessments)} energy assessments for {SURVEYORS} and {PROJECT_CODE}") + assessments_map = {} + for assessment in energy_assessments: + uploaded_xmls = list_xmls_in_s3_folder( + bucket_name=BUCKET, folder_name=os.path.join(assessment, "docs & plans") + ) + uprn = int(assessment.rstrip("/").split("/")[-1]) + assessments_map[uprn] = uploaded_xmls + + logger.info(f"Exatracted XMLS for the energy assessments") + + # For each property, we download the xmls and extract the data + for uprn, xmls in assessments_map.items(): + extracted_data = {} + for xml in xmls: + xml_data = read_from_s3(bucket_name=BUCKET, s3_file_name=xml) + xml_data_io = BytesIO(xml_data) + xml_parser = XmlParser(file=xml_data_io, filekey=xml, uprn=uprn) + xml_parser.run() + logger.info(f"Extracted data from {xml}") diff --git a/etl/xml_survey_extraction/pcdb.py b/etl/xml_survey_extraction/pcdb.py new file mode 100644 index 00000000..64d65708 --- /dev/null +++ b/etl/xml_survey_extraction/pcdb.py @@ -0,0 +1,1129 @@ +""" +This script contains the systems data, contained in the BRE product characteristics database (PCDB). + +For SAP 10.2, this can be found in the following document: +https://files.bregroup.com/SAP/SAP%2010.2%20-%2017-12-2021.pdf + +From page 157 onwards +""" +import pandas as pd + +no_heating_system = [ + { + "category": "No heating system present", + "description": "Electric heaters (assumed)", + "efficiency": 100, + "heating_type": 1, + "responsiveness": 1.0, + "code": 699 + } +] + +boiler_systems_with_radiators_or_underfloor_heating = [ + # Solid fuel boilers + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Manual feed independent boiler", + "efficiency_A": 65, + "efficiency_B": 60, + "heating_type": 2, + "responsiveness": 0.75, + "code": 151 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Auto (gravity) feed independent boiler", + "efficiency_A": 70, + "efficiency_B": 65, + "heating_type": 2, + "responsiveness": 0.75, + "code": 153 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Wood chip/pellet independent boiler", + "efficiency_A": 75, + "efficiency_B": 70, + "heating_type": 2, + "responsiveness": 0.75, + "code": 155 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Open fire with back boiler to radiators", + "efficiency_A": 63, + "efficiency_B": 55, + "heating_type": 3, + "responsiveness": 0.50, + "code": 156 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Closed room heater with boiler to radiators", + "efficiency_A": 67, + "efficiency_B": 65, + "heating_type": 3, + "responsiveness": 0.50, + "code": 158 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Stove (pellet-fired) with boiler to radiators", + "efficiency_A": 75, + "efficiency_B": 70, + "heating_type": 2, + "responsiveness": 0.75, + "code": 159 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Range cooker boiler (integral oven and boiler)", + "efficiency_A": 50, + "efficiency_B": 45, + "heating_type": 3, + "responsiveness": 0.50, + "code": 160 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Solid fuel boiler - Range cooker boiler (independent oven and boiler)", + "efficiency_A": 60, + "efficiency_B": 55, + "heating_type": 3, + "responsiveness": 0.50, + "code": 161 + }, + # Electric boilers + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - Direct acting electric boiler", + "efficiency": 100, + "heating_type": "From Table 4d", + "responsiveness": None, + "code": 191 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - CPSU in heated space – radiators or underfloor", + "efficiency": 100, + "heating_type": 1, + "responsiveness": 1.0, + "code": 192 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - Dry core storage boiler in heated space", + "efficiency": 100, + "heating_type": 2, + "responsiveness": 0.75, + "code": 193 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - Dry core storage boiler in unheated space", + "efficiency": 85, + "heating_type": 2, + "responsiveness": 0.75, + "code": 194 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - Water storage boiler in heated space", + "efficiency": 100, + "heating_type": 2, + "responsiveness": 0.75, + "code": 195 + }, + { + "category": "Boiler systems with radiators or underfloor heating", + "description": "Electric boiler - Water storage boiler in unheated space", + "efficiency": 85, + "heating_type": 2, + "responsiveness": 0.75, + "code": 196 + } +] + +heat_pumps_with_radiators_or_underfloor_heating = [ + # Electric heat pumps + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Ground source heat pump with flow temperature <= 35°C", + "space": 230, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 211 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Water source heat pump with flow temperature <= 35°C", + "space": 230, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 213 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Air source heat pump with flow temperature <= 35°C", + "space": 170, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 214 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Ground source heat pump in other cases", + "space": 170, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 221 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Water source heat pump, in other cases", + "space": 170, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 223 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Electric heat pumps - Air source heat pump in other cases", + "space": 170, + "water": 170, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 224 + }, + # Gast fired heat pumps + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Ground source heat pump with flow temperature <= 35°C", + "space": 120, + "water": 84, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 215 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Water source heat pump with flow temperature <= 35°C", + "space": 120, + "water": 84, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 216 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Air source heat pump with flow temperature <= 35°C", + "space": 110, + "water": 77, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 217 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Ground source heat pump in other cases", + "space": 84, + "water": 84, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 225 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Water source heat pump in other cases", + "space": 84, + "water": 84, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 226 + }, + { + "category": "Heat pumps with radiators or underfloor heating", + "description": "Gas-fired heat pumps - Air source heat pump in other cases", + "space": 77, + "water": 77, + "heating_type": "From Table 4d", # Replace with specific value as needed + "responsiveness": None, # Not provided, assuming 'None' + "code": 227 + } +] + +electric_heat_pumps_warm_air_distribution = [ + { + "category": "Heat pumps with warm air distribution", + "description": "Electric heat pumps - Ground source heat pump", + "space": 230, + "water": 170, + "heating_type": 1, + "responsiveness": 1.0, + "code": 521 + }, + { + "category": "Heat pumps with warm air distribution", + "description": "Electric heat pumps - Water source heat pump", + "space": 230, + "water": 170, + "heating_type": 1, + "responsiveness": 1.0, + "code": 523 + }, + { + "category": "Heat pumps with warm air distribution", + "description": "Electric heat pumps - Air source heat pump", + "space": 170, + "water": 170, + "heating_type": 1, + "responsiveness": 1.0, + "code": 524 + } +] + +gas_fired_heat_pumps_warm_air_distribution = [ + { + "category": "Heat pumps with warm air distribution", + "description": "Gas-fired heat pumps - Ground source heat pump", + "space": 120, + "water": 84, + "heating_type": 1, + "responsiveness": 1.0, + "code": 525 + }, + { + "category": "Heat pumps with warm air distribution", + "description": "Gas-fired heat pumps - Water source heat pump", + "space": 120, + "water": 84, + "heating_type": 1, + "responsiveness": 1.0, + "code": 526 + }, + { + "category": "Heat pumps with warm air distribution", + "description": "Gas-fired heat pumps - Air source heat pump", + "space": 110, + "water": 77, + "heating_type": 1, + "responsiveness": 1.0, + "code": 527 + } +] + +heat_networks = [ + { + "category": "Heat networks", + "description": "Boilers (SAP)", + "efficiency": 80, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 2 + }, + { + "category": "Heat networks", + "description": "CHP (SAP)", + "efficiency": 75, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 1 + }, + { + "category": "Heat networks", + "description": "Waste heat from power station (SAP)", + "efficiency": 100, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 4 + }, + { + "category": "Heat networks", + "description": "Heat pump (SAP)", + "efficiency": 300, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 3 + }, + { + "category": "Heat networks", + "description": "Geothermal heat source (SAP)", + "efficiency": 100, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 5 + }, + { + "category": "Heat networks", + "description": "Boilers only (RdSAP)", + "efficiency": 80, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 301 + }, + { + "category": "Heat networks", + "description": "CHP and boilers (RdSAP)", + "efficiency": 75, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 302 + }, + { + "category": "Heat networks", + "description": "Heat pump (RdSAP)", + "efficiency": 300, + "heating_type": "From table 4d", # Replace with specific value as needed + "code": 304 + } +] + +electric_storage_systems = [ + { + "category": "Electric Storage Systems", + "description": "Old (large volume) storage heaters", + "efficiency": 100, + "heating_type": 6, + "responsiveness": 0.0, + "code": 401 + }, + { + "category": "Electric Storage Systems", + "description": "Slimline storage heaters", + "code": 402, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 5, "responsiveness": 0.2}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Convector storage heaters", + "code": 403, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 5, "responsiveness": 0.2}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Fan storage heaters", + "code": 404, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Slimline storage heaters with Celect-type control", + "code": 405, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 3, "responsiveness": 0.6} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Convector storage heaters with Celect-type control", + "code": 406, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 4, "responsiveness": 0.4}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 3, "responsiveness": 0.6} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Fan storage heaters with Celect-type control", + "code": 407, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 3, "responsiveness": 0.6}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 3, "responsiveness": 0.6} + ] + }, + { + "category": "Electric Storage Systems", + "description": "Integrated storage + direct-acting heater", + "efficiency": 100, + "heating_type": 3, + "responsiveness": 0.6, + "code": 408 + }, + { + "category": "Electric Storage Systems", + "description": "High heat retention storage heaters", + "code": 409, + "options": [ + {"sub_description": "Off-peak tariffs", "efficiency": 100, "heating_type": 2, "responsiveness": 0.8}, + {"sub_description": "24-hour heating tariff", "efficiency": 100, "heating_type": 2, "responsiveness": 0.8} + ] + } +] + +off_peak_tariffs_electric_underfloor_heating = [ + { + "category": "Electric Underfloor Heating", + "description": "Off-peak tariffs - In concrete slab (off-peak only)", + "efficiency": 100, + "heating_type": 5, + "responsiveness": 0.0, + "code": 421 + }, + { + "category": "Electric Underfloor Heating", + "description": "Off-peak tariffs - Integrated (storage+direct-acting)", + "efficiency": 100, + "heating_type": 4, + "responsiveness": 0.25, + "code": 422 + }, + { + "category": "Electric Underfloor Heating", + "description": "Off-peak tariffs - Integrated (storage+direct-acting) with low (off-peak) tariff control", + "efficiency": 100, + "heating_type": 3, + "responsiveness": 0.50, + "code": 423 + } +] + +standard_or_off_peak_tariff_electric_underfloor_heating = [ + { + "category": "Electric Underfloor Heating", + "description": "Standard or off-peak tariff - In screed above insulation", + "efficiency": 100, + "heating_type": 2, + "responsiveness": 0.75, + "code": 424 + }, + { + "category": "Electric Underfloor Heating", + "description": "Standard or off-peak tariff - In timber floor, or immediately below floor covering", + "efficiency": 100, + "heating_type": 1, + "responsiveness": 1.0, + "code": 425 + } +] + +gas_fired_warm_air_fan_assisted = [ + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Ducted, on-off control, pre 1998", "efficiency": 70, + "heating_type": 1, + "responsiveness": 1.0, + "code": 501 + }, + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Ducted, on-off control, 1998 or later", + "efficiency": 76, + "heating_type": 1, + "responsiveness": 1.0, + "code": 502 + }, + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Ducted, modulating control, pre 1998", + "efficiency": 72, + "heating_type": 1, + "responsiveness": 1.0, + "code": 503 + }, + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Ducted, modulating control, 1998 or later", + "efficiency": 78, + "heating_type": 1, + "responsiveness": 1.0, + "code": 504 + }, + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Room heater with in-floor ducts", + "efficiency": 69, + "heating_type": 1, + "responsiveness": 1.0, + "code": 505 + }, + { + "category": "Warm Air Systems", + "description": "Gas-fired warm air with fan-assisted flue - Condensing", + "efficiency": 81, + "heating_type": 1, + "responsiveness": 1.0, + "code": 520 + } +] + +gas_fired_warm_air_balanced_or_open_flue = [ + {"category": "Warm Air Systems", + "description": "Gas-fired warm air with balanced or open flue - Ducted or stub-ducted, on-off control, pre 1998", + "efficiency": 70, "heating_type": 1, "responsiveness": 1.0, "code": 506}, + {"category": "Warm Air Systems", + "description": "Gas-fired warm air with balanced or open flue - Ducted or stub-ducted, on-off control, " + "1998 or later", + "efficiency": 76, "heating_type": 1, "responsiveness": 1.0, "code": 507}, + {"category": "Warm Air Systems", + "description": "Gas-fired warm air with balanced or open flue - Ducted or stub-ducted, modulating control, " + "pre 1998", + "efficiency": 72, "heating_type": 1, "responsiveness": 1.0, "code": 508}, + {"category": "Warm Air Systems", + "description": "Gas-fired warm air with balanced or open flue - Ducted or stub-ducted, modulating control, " + "1998 or later", + "efficiency": 78, "heating_type": 1, "responsiveness": 1.0, "code": 509}, + {"category": "Warm Air Systems", + "description": "Gas-fired warm air with balanced or open flue - Ducted or stub-ducted with flue heat recovery", + "efficiency": 85, "heating_type": 1, "responsiveness": 1.0, "code": 510}, + {"category": "Warm Air Systems", "description": "Gas-fired warm air with balanced or open flue - Condensing", + "efficiency": 81, "heating_type": 1, "responsiveness": 1.0, "code": 511} +] + +liquid_fired_warm_air = [ + {"category": "Warm Air Systems", "description": "Liquid-fired warm air - Ducted output (on/off control)", + "efficiency": 70, "heating_type": 1, "responsiveness": 1.0, "code": 512}, + {"category": "Warm Air Systems", "description": "Liquid-fired warm air - Ducted output (modulating control)", + "efficiency": 72, "heating_type": 1, "responsiveness": 1.0, "code": 513}, + {"category": "Warm Air Systems", "description": "Liquid-fired warm air - Stub duct system", "efficiency": 70, + "heating_type": 1, "responsiveness": 1.0, "code": 514} +] + +electric_warm_air_systems = [ + { + "category": "Warm Air Systems", + "description": "Electric warm air - Electricaire system", + "efficiency": 100, + "heating_type": 2, + "responsiveness": 0.75, + "code": 515 + } +] + +room_heaters = [ + # Gas (including LPG and biogas) room heaters + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire, open flue, pre-1980 (open fronted)", + "flue": "OF", "efficiency_A": 50, "efficiency_B": 50, "heating_type": 1, "responsiveness": 1.0, "code": 601}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire, open flue, pre-1980 (open fronted), " + "with back boiler unit", + "flue": "OF*", "efficiency_A": 50, "efficiency_B": 50, "heating_type": 1, "responsiveness": 1.0, "code": 602}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire, open flue, 1980 or later (open fronted), " + "sitting proud of, and sealed to, fireplace opening", + "flue": "OF", "efficiency_A": 63, "efficiency_B": 64, "heating_type": 1, "responsiveness": 1.0, "code": 603}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire, open flue, 1980 or later (open fronted), " + "sitting proud of, and sealed to, fireplace opening, with back boiler unit", + "flue": "OF*", "efficiency_A": 63, "efficiency_B": 64, "heating_type": 1, "responsiveness": 1.0, "code": 604}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Flush fitting Live Fuel Effect gas fire (open " + "fronted), sealed to fireplace opening", + "flue": "OF", "efficiency_A": 40, "efficiency_B": 41, "heating_type": 1, "responsiveness": 1.0, "code": 605}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Flush fitting Live Fuel Effect gas fire (open " + "fronted), sealed to fireplace opening, with back boiler unit", + "flue": "OF*", "efficiency_A": 40, "efficiency_B": 41, "heating_type": 1, "responsiveness": 1.0, "code": 606}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Flush fitting Live Fuel Effect gas fire (open " + "fronted), fan assisted, sealed to fireplace opening", + "flue": "OF", "efficiency_A": 45, "efficiency_B": 46, "heating_type": 1, "responsiveness": 1.0, "code": 607}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire or wall heater, balanced flue", + "flue": "RS", "efficiency_A": 58, "efficiency_B": 60, "heating_type": 1, "responsiveness": 1.0, "code": 609}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Gas fire, closed fronted, fan assisted", + "flue": "RS", "efficiency_A": 72, "efficiency_B": 73, "heating_type": 1, "responsiveness": 1.0, "code": 610}, + {"category": "Room Heaters", "description": "Gas (including LPG and biogas) room heaters - Condensing gas fire", + "flue": "RS", "efficiency_A": 85, "efficiency_B": 85, "heating_type": 1, "responsiveness": 1.0, "code": 611}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Decorative Fuel Effect gas fire, open to chimney", + "flue": "C", "efficiency_A": 20, "efficiency_B": 20, "heating_type": 1, "responsiveness": 1.0, "code": 612}, + {"category": "Room Heaters", + "description": "Gas (including LPG and biogas) room heaters - Flueless gas fire, secondary heating only", + "flue": "none", "efficiency_A": 90, "efficiency_B": 92, "heating_type": 1, "responsiveness": 1.0, "code": 613}, + + # Liquid fuel room heaters + {"category": "Room Heaters", "description": "Liquid fuel room heaters - Room heater, pre 2000", "efficiency": 55, + "heating_type": 1, "responsiveness": 1.0, "code": 621}, + {"category": "Room Heaters", + "description": "Liquid fuel room heaters - Room heater, pre 2000, with boiler (no radiators)", "efficiency": 65, + "heating_type": 1, "responsiveness": 1.0, "code": 622}, + {"category": "Room Heaters", "description": "Liquid fuel room heaters - Room heater, 2000 or later", + "efficiency": 60, "heating_type": 1, "responsiveness": 1.0, "code": 623}, + {"category": "Room Heaters", + "description": "Liquid fuel room heaters - Room heater, 2000 or later with boiler (no radiators)", + "efficiency": 70, "heating_type": 1, "responsiveness": 1.0, "code": 624}, + {"category": "Room Heaters", "description": "Liquid fuel room heaters - Bioethanol heater, secondary heating only", + "efficiency": 94, "heating_type": 1, "responsiveness": 1.0, "code": 625}, + + # Solid fuel room heaters + {"category": "Room Heaters", "description": "Solid fuel room heaters - Open fire in grate", "efficiency_A": 37, + "efficiency_B": 32, "heating_type": 3, "responsiveness": 0.5, "code": 631}, + {"category": "Room Heaters", "description": "Solid fuel room heaters - Open fire with back boiler (no radiators)", + "efficiency_A": 50, "efficiency_B": 50, "heating_type": 3, "responsiveness": 0.5, "code": 632}, + {"category": "Room Heaters", "description": "Solid fuel room heaters - Closed room heater", "efficiency_A": 65, + "efficiency_B": 60, "heating_type": 3, "responsiveness": 0.5, "code": 633}, + {"category": "Room Heaters", + "description": "Solid fuel room heaters - Closed room heater with boiler (no radiators)", "efficiency_A": 67, + "efficiency_B": 65, "heating_type": 3, "responsiveness": 0.5, "code": 634}, + {"category": "Room Heaters", "description": "Solid fuel room heaters - Stove (pellet fired)", "efficiency_A": 70, + "efficiency_B": 65, "heating_type": 2, "responsiveness": 0.75, "code": 635}, + {"category": "Room Heaters", + "description": "Solid fuel room heaters - Stove (pellet fired) with boiler (no radiators)", "efficiency_A": 75, + "efficiency_B": 70, "heating_type": 2, "responsiveness": 0.75, "code": 636}, + + # Electric (direct acting) room heaters + {"category": "Room Heaters", + "description": "Electric (direct acting) room heaters - Panel, convector or radiant heaters", "efficiency": 100, + "heating_type": 1, "responsiveness": 1.0, "code": 691}, + {"category": "Room Heaters", + "description": "Electric (direct acting) room heaters - Water- or oil-filled radiators", "efficiency": 100, + "heating_type": 1, "responsiveness": 1.0, "code": 694}, + {"category": "Room Heaters", "description": "Electric (direct acting) room heaters - Fan heaters", + "efficiency": 100, "heating_type": 1, "responsiveness": 1.0, "code": 692}, + {"category": "Room Heaters", "description": "Electric (direct acting) room heaters - Portable electric heaters", + "efficiency": 100, "heating_type": 1, "responsiveness": 1.0, "code": 693} +] + +other_space_heating_systems = [ + { + "category": "Other Space Heating Systems", + "description": "Electric ceiling heating", + "efficiency": 100, + "heating_type": 2, + "responsiveness": 0.75, + "code": 701 + } +] + +hot_water_systems = [ + {"category": "Hot Water Systems", "description": "No hot water system present - electric immersion assumed", + "efficiency": 100, "code": 999}, + { + "category": "Hot Water Systems", + "description": "HWP from the primary heating system", + "code": 901, + "options": [ + {"sub_description": "Back boiler (hot water only), gas*", "efficiency": 65}, + {"sub_description": "Circulator built into a gas warm air system, pre 1998", "efficiency": 65}, + {"sub_description": "Circulator built into a gas warm air system, 1998 or later", "efficiency": 73}, + {"sub_description": "Heat exchanger in a gas warm air system, condensing unit", "efficiency": 74}, + ] + }, + {"category": "Hot Water Systems", + "description": "From second main system", "efficiency": None, + "code": 914}, + {"category": "Hot Water Systems", "description": "From secondary system", + "efficiency": None, "code": 902}, + {"category": "Hot Water Systems", "description": "Electric immersion", "efficiency": 100, "code": 903}, + {"category": "Hot Water Systems", + "description": "Single-point gas-fired water heater (instantaneous at point of use)", "efficiency": 70, + "code": 907}, + {"category": "Hot Water Systems", + "description": "Multi-point gas-fired water heater (instantaneous serving several taps)", "efficiency": 65, + "code": 908}, + {"category": "Hot Water Systems", "description": "Electric instantaneous at point of use", "efficiency": 100, + "code": 909}, + {"category": "Hot Water Systems", "description": "Gas boiler/circulator for water heating only*", "efficiency": 65, + "code": 911}, + {"category": "Hot Water Systems", "description": "Liquid fuel boiler/circulator for water heating only*", + "efficiency": 70, "code": 912}, + {"category": "Hot Water Systems", "description": "Solid fuel boiler/circulator for water heating only", + "efficiency": 55, "code": 913}, + # Range cookers with boiler for water heating only + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, single burner with permanent pilot", + "efficiency": 46, "code": 921}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, single burner with automatic ignition", + "efficiency": 50, + "code": 922}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, twin burner with permanent pilot pre 1998", + "efficiency": 60, + "code": 923}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, twin burner with automatic ignition pre " + "1998", + "efficiency": 65, "code": 924}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, twin burner with permanent pilot 1998 or " + "later", + "efficiency": 65, "code": 925}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Gas, twin burner with automatic ignition 1998 " + "or later", + "efficiency": 70, "code": 926}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Liquid fuel, single burner", "efficiency": 60, + "code": 927}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Liquid fuel, twin burner pre 1998", + "efficiency": 70, + "code": 928}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Liquid fuel, twin burner 1998 or later", + "efficiency": 75, + "code": 929}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Solid fuel, integral oven and boiler", + "efficiency": 45, + "code": 930}, + {"category": "Hot Water Systems", + "description": "Range cooker with boiler for water heating only: Solid fuel, independent oven and boiler", + "efficiency": 55, + "code": 931}, + # Electric heat pump for water heating only + {"category": "Hot Water Systems", "description": "Electric heat pump for water heating only*", "efficiency": 170, + "code": 941}, + # Hot-water only heat network + # Remove the SAP version + # {"category": "Hot Water Systems", + # "description": "Hot-water only heat network (SAP)", "efficiency": None, + # "code": 950}, + {"category": "Hot Water Systems", "description": "Hot-water only heat network (RdSAP) - boilers", "efficiency": 80, + "code": 950}, + {"category": "Hot Water Systems", "description": "Hot-water only heat network (RdSAP) - CHP", "efficiency": 75, + "code": 951}, + {"category": "Hot Water Systems", "description": "Hot-water only heat network (RdSAP) - heat pump", + "efficiency": 300, "code": 952} +] + +boilers_seasonal = [ + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Regular non-condensing with " + "automatic ignition", + "efficiency_winter": 74, "efficiency_summer": 64, "code": 101}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Regular condensing with " + "automatic ignition", + "efficiency_winter": 84, "efficiency_summer": 74, "code": 102}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Non-condensing combi with " + "automatic ignition", + "efficiency_winter": 74, "efficiency_summer": 65, "code": 103}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Condensing combi with " + "automatic ignition", + "efficiency_winter": 84, "efficiency_summer": 75, "code": 104}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Regular non-condensing with " + "permanent pilot light", + "efficiency_winter": 70, "efficiency_summer": 60, "code": 105}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Regular condensing with " + "permanent pilot light", + "efficiency_winter": 80, "efficiency_summer": 70, "code": 106}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Non-condensing combi with " + "permanent pilot light", + "efficiency_winter": 70, "efficiency_summer": 61, "code": 107}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Condensing combi with " + "permanent pilot light", + "efficiency_winter": 80, "efficiency_summer": 71, "code": 108}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) 1998 or later - Back boiler to radiators", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 109}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with fan-assisted flue - Regular, " + "low thermal capacity", + "efficiency_winter": 73, "efficiency_summer": 63, "code": 110}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with fan-assisted flue - Regular, " + "high or unknown thermal capacity", + "efficiency_winter": 69, "efficiency_summer": 59, "code": 111}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with fan-assisted flue - Combi", + "efficiency_winter": 71, "efficiency_summer": 62, "code": 112}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with fan-assisted flue - Condensing " + "combi", + "efficiency_winter": 84, "efficiency_summer": 75, "code": 113}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with fan-assisted flue - Regular, " + "condensing", + "efficiency_winter": 84, "efficiency_summer": 74, "code": 114}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with balanced or open flue - " + "Regular, wall mounted", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 115}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with balanced or open flue - " + "Regular, floor mounted, pre 1979", + "efficiency_winter": 56, "efficiency_summer": 46, "code": 116}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with balanced or open flue - " + "Regular, floor mounted, 1979 to 1997", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 117}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with balanced or open flue - Combi", + "efficiency_winter": 66, "efficiency_summer": 57, "code": 118}, + {"category": "Boilers - seasonal", + "description": "Gas boilers (including mains gas, LPG and biogas) pre-1998, with balanced or open flue - Back " + "boiler to radiators", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 119}, + {"category": "Boilers - seasonal", + "description": "Combined Primary Storage Units (CPSU) (mains gas, LPG and biogas) - With automatic ignition (" + "non-condensing)", + "efficiency_winter": 74, "efficiency_summer": 72, "code": 120}, + {"category": "Boilers - seasonal", + "description": "Combined Primary Storage Units (CPSU) (mains gas, LPG and biogas) - With automatic ignition (" + "condensing)", + "efficiency_winter": 83, "efficiency_summer": 81, "code": 121}, + {"category": "Boilers - seasonal", + "description": "Combined Primary Storage Units (CPSU) (mains gas, LPG and biogas) - With permanent pilot (" + "non-condensing)", + "efficiency_winter": 70, "efficiency_summer": 68, "code": 122}, + {"category": "Boilers - seasonal", + "description": "Combined Primary Storage Units (CPSU) (mains gas, LPG and biogas) - With permanent pilot (" + "condensing)", + "efficiency_winter": 79, "efficiency_summer": 77, "code": 123}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Standard oil boiler pre-1985", + "efficiency_winter": 66, "efficiency_summer": 54, "code": 124}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Standard oil boiler 1985 to 1997", + "efficiency_winter": 71, "efficiency_summer": 59, "code": 125}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Standard oil boiler, 1998 or later", + "efficiency_winter": 80, "efficiency_summer": 68, "code": 126}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Condensing oil boiler", + "efficiency_winter": 84, "efficiency_summer": 72, "code": 127}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Combi oil boiler, pre-1998", + "efficiency_winter": 71, "efficiency_summer": 62, "code": 128}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Combi oil boiler, 1998 or later", + "efficiency_winter": 77, "efficiency_summer": 68, "code": 129}, + {"category": "Boilers - seasonal", "description": "Liquid fuel boilers - Condensing combi oil boiler", + "efficiency_winter": 82, "efficiency_summer": 73, "code": 130}, + {"category": "Boilers - seasonal", + "description": "Liquid fuel boilers - Oil room heater with boiler to radiators, pre 2000", "efficiency_winter": 66, + "efficiency_summer": 54, "code": 131}, + {"category": "Boilers - seasonal", + "description": "Liquid fuel boilers - Oil room heater with boiler to radiators, 2000 or later", + "efficiency_winter": 71, "efficiency_summer": 59, "code": 132}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Single burner with permanent pilot", + "efficiency_winter": 47, "efficiency_summer": 37, "code": 133}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Single burner with automatic ignition", + "efficiency_winter": 51, "efficiency_summer": 41, "code": 134}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Twin burner with permanent pilot (" + "non-condensing) pre 1998", + "efficiency_winter": 61, "efficiency_summer": 51, "code": 135}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Twin burner with automatic ignition (" + "non-condensing) pre 1998", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 136}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Twin burner with permanent pilot (" + "non-condensing) 1998 or later", + "efficiency_winter": 66, "efficiency_summer": 56, "code": 137}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (mains gas, LPG and biogas) - Twin burner with automatic ignition (" + "non-condensing) 1998 or later", + "efficiency_winter": 71, "efficiency_summer": 61, "code": 138}, + {"category": "Boilers - seasonal", "description": "Range cooker boilers (liquid fuel) - Single burner", + "efficiency_winter": 61, "efficiency_summer": 49, "code": 139}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (liquid fuel) - Twin burner (non-condensing) pre 1998", + "efficiency_winter": 71, "efficiency_summer": 59, "code": 140}, + {"category": "Boilers - seasonal", + "description": "Range cooker boilers (liquid fuel) - Twin burner (non-condensing) 1998 or later", + "efficiency_winter": 76, "efficiency_summer": 64, "code": 141}, +] + +# Heating controls +no_heating_system_controls = [ + { + "category": "No heating system present", + "description": "None", + "control": 2, + "temperature_adjustment_c": "+0.3", + "code": 2699 + } +] + +boiler_system_controls = [ + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Not applicable", "control": None, "temperature_adjustment_c": None, + "code": 2100}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "No time or thermostatic control of room temperature", "control": 1, + "temperature_adjustment_c": "+0.6", "code": 2101}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "Programmer, no room thermostat", + "control": 1, "temperature_adjustment_c": "+0.6", "code": 2102}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "Room thermostat only", + "control": 1, "temperature_adjustment_c": "0", "code": 2103}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "Programmer and room thermostat", + "control": 1, "temperature_adjustment_c": "0", "code": 2104}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Programmer and at least two room thermostats", "control": 2, "temperature_adjustment_c": "0", + "code": 2105}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "Room thermostat and TRVs", + "control": 2, "temperature_adjustment_c": "0", "code": 2113}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Programmer, room thermostat and TRVs", "control": 2, "temperature_adjustment_c": "0", + "code": 2106}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "TRVs and bypass", "control": 2, + "temperature_adjustment_c": "0", "code": 2111}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", "description": "Programmer, TRVs and bypass", + "control": 2, "temperature_adjustment_c": "0", "code": 2107}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Programmer, TRVs and flow switch", "control": 2, "temperature_adjustment_c": "0", "code": 2108}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Programmer, TRVs and boiler energy manager", "control": 2, "temperature_adjustment_c": "0", + "code": 2109}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Time and temperature zone control by arrangement of plumbing and electrical services", + "control": 3, "temperature_adjustment_c": "0", "code": 2110}, + {"category": "Boiler Systems with Radiators or Underfloor Heating", + "description": "Time and temperature zone control by device in PCDB", "control": 3, + "temperature_adjustment_c": "0", "code": 2112}, +] + +heat_pump_controls = [ + # We have a previous 2100 code for not applicable + # {"category": "Heat Pumps with Radiators or Underfloor Heating", + # "description": "Not applicable (heat pump provides DHW only)", "control": None, "temperature_adjustment_c": None, + # "code": 2100}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", + "description": "No time or thermostatic control of room temperature", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2201}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", "description": "Programmer, no room thermostat", + "control": 1, "temperature_adjustment_c": "+0.3", "code": 2202}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", "description": "Room thermostat only", "control": 1, + "temperature_adjustment_c": "0", "code": 2203}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", "description": "Programmer and room thermostat", + "control": 1, "temperature_adjustment_c": "0", "code": 2204}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", + "description": "Programmer and at least two room thermostats", "control": 2, "temperature_adjustment_c": "0", + "code": 2205}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", "description": "Room thermostat and TRVs", + "control": 2, "temperature_adjustment_c": "0", "code": 2209}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", + "description": "Programmer, room thermostat and TRVs", "control": 2, "temperature_adjustment_c": "0", + "code": 2210}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", "description": "Programmer, TRVs and bypass", + "control": 2, "temperature_adjustment_c": "0", "code": 2206}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", + "description": "Time and temperature zone control by arrangement of plumbing and electrical services", + "control": 3, "temperature_adjustment_c": "0", "code": 2207}, + {"category": "Heat Pumps with Radiators or Underfloor Heating", + "description": "Time and temperature zone control by device in PCDB", "control": 3, + "temperature_adjustment_c": "0", "code": 2208}, +] + +heat_network_controls = [ + {"category": "Heat Networks", "description": "Flat rate charging*, no thermostatic control of room temperature", + "control": 1, "temperature_adjustment_c": "+0.3", "code": 2301}, + {"category": "Heat Networks", "description": "Flat rate charging*, programmer, no room thermostat", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2302}, + {"category": "Heat Networks", "description": "Flat rate charging*, room thermostat only", "control": 1, + "temperature_adjustment_c": "0", "code": 2303}, + {"category": "Heat Networks", "description": "Flat rate charging*, programmer and room thermostat", "control": 1, + "temperature_adjustment_c": "0", "code": 2304}, + {"category": "Heat Networks", "description": "Flat rate charging*, room thermostat and TRVs", "control": 2, + "temperature_adjustment_c": "0", "code": 2313}, + {"category": "Heat Networks", "description": "Flat rate charging*, TRVs", "control": 2, + "temperature_adjustment_c": "0", "code": 2307}, + {"category": "Heat Networks", "description": "Flat rate charging*, programmer and TRVs", "control": 2, + "temperature_adjustment_c": "0", "code": 2305}, + {"category": "Heat Networks", "description": "Flat rate charging*, programmer and at least two room thermostats", + "control": 2, "temperature_adjustment_c": "0", "code": 2311}, + {"category": "Heat Networks", "description": "Charging system linked to use of heating, room thermostat only", + "control": 2, "temperature_adjustment_c": "0", "code": 2308}, + {"category": "Heat Networks", + "description": "Charging system linked to use of heating, programmer and room thermostat", "control": 2, + "temperature_adjustment_c": "0", "code": 2309}, + {"category": "Heat Networks", "description": "Charging system linked to use of heating, room thermostat and TRVs", + "control": 3, "temperature_adjustment_c": "0", "code": 2314}, + {"category": "Heat Networks", "description": "Charging system linked to use of heating, TRVs", "control": 3, + "temperature_adjustment_c": "0", "code": 2310}, + {"category": "Heat Networks", "description": "Charging system linked to use of heating, programmer and TRVs", + "control": 3, "temperature_adjustment_c": "0", "code": 2306}, + {"category": "Heat Networks", + "description": "Charging system linked to use of heating, programmer and at least two room thermostats", + "control": 3, "temperature_adjustment_c": "0", "code": 2312}, +] + +electric_storage_systems_controls = [ + {"category": "Electric Storage Systems", "description": "Manual charge control", "control": 3, + "temperature_adjustment_c": "+0.7", "code": 2401}, + {"category": "Electric Storage Systems", "description": "Automatic charge control", "control": 3, + "temperature_adjustment_c": "+0.4", "code": 2402}, + {"category": "Electric Storage Systems", "description": "Celect-type controls", "control": 3, + "temperature_adjustment_c": "+0.4", "code": 2403}, + {"category": "Electric Storage Systems", "description": "Controls for high heat retention storage heaters §", + "control": 3, "temperature_adjustment_c": "0", "code": 2404}, +] + +warm_air_systems_controls = [ + {"category": "Warm Air Systems", "description": "No time or thermostatic control of room temperature", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2501}, + {"category": "Warm Air Systems", "description": "Programmer, no room thermostat", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2502}, + {"category": "Warm Air Systems", "description": "Room thermostat only", "control": 1, + "temperature_adjustment_c": "0", "code": 2503}, + {"category": "Warm Air Systems", "description": "Programmer and room thermostat", "control": 1, + "temperature_adjustment_c": "0", "code": 2504}, + {"category": "Warm Air Systems", "description": "Programmer and at least two room thermostats", "control": 2, + "temperature_adjustment_c": "0", "code": 2505}, + {"category": "Warm Air Systems", "description": "Time and temperature zone control", "control": 3, + "temperature_adjustment_c": "0", "code": 2506}, +] + +room_heater_systems_controls = [ + {"category": "Room Heater Systems", "description": "No thermostatic control of room temperature", "control": 2, + "temperature_adjustment_c": "+0.3", "code": 2601}, + {"category": "Room Heater Systems", "description": "Appliance thermostats", "control": 3, + "temperature_adjustment_c": "0", "code": 2602}, + {"category": "Room Heater Systems", "description": "Programmer and appliance thermostats", "control": 3, + "temperature_adjustment_c": "0", "code": 2603}, + {"category": "Room Heater Systems", "description": "Room thermostats only", "control": 3, + "temperature_adjustment_c": "0", "code": 2604}, + {"category": "Room Heater Systems", "description": "Programmer and room thermostats", "control": 3, + "temperature_adjustment_c": "0", "code": 2605}, +] + +other_systems_controls = [ + {"category": "Other Systems", "description": "No time or thermostatic control of room temperature", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2701}, + {"category": "Other Systems", "description": "Programmer, no room thermostat", "control": 1, + "temperature_adjustment_c": "+0.3", "code": 2702}, + {"category": "Other Systems", "description": "Room thermostat only", "control": 1, "temperature_adjustment_c": "0", + "code": 2703}, + {"category": "Other Systems", "description": "Programmer and room thermostat", "control": 1, + "temperature_adjustment_c": "0", "code": 2704}, + {"category": "Other Systems", "description": "Temperature zone control", "control": 2, + "temperature_adjustment_c": "0", "code": 2705}, + {"category": "Other Systems", "description": "Time and temperature zone control", "control": 3, + "temperature_adjustment_c": "0", "code": 2706}, +] + +heating_data = ( + no_heating_system + + boiler_systems_with_radiators_or_underfloor_heating + + heat_pumps_with_radiators_or_underfloor_heating + + electric_heat_pumps_warm_air_distribution + + gas_fired_heat_pumps_warm_air_distribution + + heat_networks + + electric_storage_systems + + off_peak_tariffs_electric_underfloor_heating + + standard_or_off_peak_tariff_electric_underfloor_heating + + gas_fired_warm_air_fan_assisted + + gas_fired_warm_air_balanced_or_open_flue + + liquid_fired_warm_air + + electric_warm_air_systems + + room_heaters + + other_space_heating_systems + + hot_water_systems + + boilers_seasonal + + no_heating_system_controls + + boiler_system_controls + + heat_pump_controls + + heat_network_controls + + electric_storage_systems_controls + + warm_air_systems_controls + + room_heater_systems_controls + + other_systems_controls +) + +heating_data = pd.DataFrame(heating_data) diff --git a/utils/s3.py b/utils/s3.py index 1b14ca97..b3553824 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -276,3 +276,86 @@ def list_files_in_s3_folder(bucket_name, folder_name): except Exception as e: logger.error(f'Failed to list files in folder {folder_name} in bucket {bucket_name}: {str(e)}') return [] + + +def list_files_and_subfolders_in_s3_folder(bucket_name, folder_name): + """ + List all files and immediate subfolders in a given folder in an S3 bucket. + + E.g. if we have a folder structure in S3 like this: + - folder1/ + - file1.csv + - file2.csv + - subfolder1/ + - file3.csv + + Then calling list_files_and_subfolders_in_s3_folder(bucket_name='my-bucket', folder_name='folder1/') + would return ['folder1/file1.csv', 'folder1/file2.csv', 'folder1/subfolder1/']. + + Namely, the nested files are not included in the list, only the immediate files and subfolders. + + :param bucket_name: The name of the S3 bucket. + :param folder_name: The folder name within the S3 bucket. + :return: A list of file keys and subfolder prefixes in the specified S3 folder. + """ + + # For this function, folder_name should end with a forward slash + if not folder_name.endswith('/'): + folder_name += '/' + + try: + s3 = boto3.client('s3') + response = s3.list_objects_v2(Bucket=bucket_name, Prefix=folder_name, Delimiter='/') + + items = [] + + # Add files to the list + if 'Contents' in response: + items.extend([content['Key'] for content in response['Contents'] if content['Key'] != folder_name]) + + # Add immediate subfolders to the list + if 'CommonPrefixes' in response: + items.extend([prefix['Prefix'] for prefix in response['CommonPrefixes']]) + + return items + + except NoCredentialsError: + logger.error("Credentials not available.") + return [] + except PartialCredentialsError: + logger.error("Incomplete credentials provided.") + return [] + except Exception as e: + logger.error(f'Failed to list files and subfolders in folder {folder_name} in bucket {bucket_name}: {str(e)}') + return [] + + +def list_xmls_in_s3_folder(bucket_name, folder_name): + """ + List all XML files in a given folder in an S3 bucket. + + :param bucket_name: The name of the S3 bucket. + :param folder_name: The folder name within the S3 bucket. + :return: A list of XML file keys in the specified S3 folder. + """ + try: + s3 = boto3.client('s3') + response = s3.list_objects_v2(Bucket=bucket_name, Prefix=folder_name) + + if 'Contents' not in response: + logger.info(f"No files found in folder {folder_name} in bucket {bucket_name}.") + return [] + + # Filter XML files + xml_files = [content['Key'] for content in response['Contents'] if content['Key'].endswith('.xml')] + return xml_files + + except NoCredentialsError: + logger.error("Credentials not available.") + return [] + except PartialCredentialsError: + logger.error("Incomplete credentials provided.") + return [] + except Exception as e: + logger.error(f'Failed to list XML files in folder {folder_name} in bucket {bucket_name}: {str(e)}') + return []