From 723beaf1041168461220da87ebdee8477d9c3f8c Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 26 Aug 2025 10:44:31 +0100 Subject: [PATCH] fix pipeline for July 2025 data, keep lodgement date to do analysis on new rdsap standard in ML stage --- .gitignore | 4 +- BaseUtility.py | 21 +- etl/epc/DataProcessor.py | 8 +- etl/epc/Dataset.py | 39 +- recommendations/rdsap_tables.py | 864 +++++++++++++++++++++++++++----- 5 files changed, 775 insertions(+), 161 deletions(-) diff --git a/.gitignore b/.gitignore index 5e247d77..a6538116 100644 --- a/.gitignore +++ b/.gitignore @@ -275,4 +275,6 @@ cache/ */.idea *.png -*.pptx \ No newline at end of file +*.pptx + +local_data* \ No newline at end of file diff --git a/BaseUtility.py b/BaseUtility.py index e799144d..2f990695 100644 --- a/BaseUtility.py +++ b/BaseUtility.py @@ -30,24 +30,25 @@ class Definitions: # The Building Emission Rate (BER) data field for non-domestic buildings may contain a ‘blank’ value. The BER # was only lodged on the register from 7 March 2010. "Blank" - # There are currently just over 8,600 records where the local authority identifier is ‘null’. This is due to - # the Register Operator not being able to match the building address in the Markermap Ordinance Survey (GB) - # lookup tables or OS MasterMap Address Layer 2 data. The majority of these addresses have been requested - # manually by energy assessors for inclusion by the Register Operator in the registers (e.g. new builds, - # etc). These records are being published for completeness. An ongoing process to manage these manually added + # There are currently just over 8,600 records where the local authority identifier is ‘null’. This is due to + # the Register Operator not being able to match the building address in the Markermap Ordinance Survey (GB) + # lookup tables or OS MasterMap Address Layer 2 data. The majority of these addresses have been requested + # manually by energy assessors for inclusion by the Register Operator in the registers (e.g. new builds, + # etc). These records are being published for completeness. An ongoing process to manage these manually added # addresses will take time to develop to deal with these and future anomalies. # - # There are several fields within the lodged data where it is possible to enter multiple entries to cater for - # different data_types of build within a single property, i.e. extensions. This results in multiple entries for - # the description fields for floor, roof and wall. For the purposes of this data release only the information - # contained within the first of these multiple entries is being provided. As there are no restrictions on the - # value in this first field it means that sometimes the first field in a multiple entry description field may + # There are several fields within the lodged data where it is possible to enter multiple entries to cater for + # different data_types of build within a single property, i.e. extensions. This results in multiple entries for + # the description fields for floor, roof and wall. For the purposes of this data release only the information + # contained within the first of these multiple entries is being provided. As there are no restrictions on the + # value in this first field it means that sometimes the first field in a multiple entry description field may # contain a ‘null’ value. A resolution to correct these anomalies will be considered for future data releases. "NULL", # We sometimes see fields populated with just an empty string. "", # An older value which rarely shows up but has been seen in the data. "UNKNOWN", + "Unknown", } DATA_ANOMALY_SUBSTRINGS = { diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 9655cf77..99987f48 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -48,6 +48,8 @@ construction_age_bounds_map = { "England and Wales: 2003-2006": {"l": 2003, "u": 2006}, "England and Wales: 2007-2011": {"l": 2007, "u": 2011}, "England and Wales: 2012 onwards": {"l": 2012, "u": 3000}, + "England and Wales: 2012-2021": {"l": 2012, "u": 2021}, + "England and Wales: 2022 onwards": {"l": 2022, "u": 3000}, } construction_age_remap = { @@ -384,7 +386,7 @@ class EPCDataProcessor: has_missings = pd.isnull(self.data[col]).sum() while has_missings: self.data = apply_clean( - data=self.data, matching_columns=matching_columns[0: to_index + 1] + data=self.data, matching_columns=matching_columns[0 : to_index + 1] ) has_missings = pd.isnull(self.data[col]).sum() @@ -858,7 +860,9 @@ class EPCDataProcessor: # Fill NaN values with averages for col in cols_to_clean: - data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"]) + data_to_clean[col] = data_to_clean[col].fillna( + data_to_clean[f"{col}_AVERAGE"] + ) data_to_clean = data_to_clean.drop(columns=[f"{col}_AVERAGE"]) # If we still have missings data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[col].mean()) diff --git a/etl/epc/Dataset.py b/etl/epc/Dataset.py index 5d3720fc..35bc108e 100644 --- a/etl/epc/Dataset.py +++ b/etl/epc/Dataset.py @@ -8,7 +8,9 @@ from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes -from etl.epc_clean.epc_attributes.MainheatControlAttributes import MainheatControlAttributes +from etl.epc_clean.epc_attributes.MainheatControlAttributes import ( + MainheatControlAttributes, +) from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes from etl.epc_clean.epc_attributes.MainFuelAttributes import MainFuelAttributes @@ -169,7 +171,7 @@ class TrainingDataset(BaseDataset): self.df = pd.DataFrame([dataset.difference_record for dataset in datasets]) self._feature_generation() - self._drop_features() + # self._drop_features() self._clean_efficiency_variables() self._null_validation(information="Clean Efficiency Variables") self._expand_description_to_features(cleaned_lookup) @@ -210,11 +212,11 @@ class TrainingDataset(BaseDataset): common_cols = [[col + "_starting", col + "_ending"] for col in common_cols] self.df = self.df.loc[ - :, - no_suffix_cols - + only_ending_cols - + [col for cols in common_cols for col in cols], - ] + :, + no_suffix_cols + + only_ending_cols + + [col for cols in common_cols for col in cols], + ] def _remove_abnormal_change_in_floor_area(self): """ @@ -519,7 +521,7 @@ class TrainingDataset(BaseDataset): expanded_df["is_sandstone_or_limestone"] == expanded_df["is_sandstone_or_limestone_ending"] ) - ] + ] elif component == "floor": expanded_df = expanded_df[ (expanded_df["is_suspended"] == expanded_df["is_suspended_ending"]) @@ -536,7 +538,7 @@ class TrainingDataset(BaseDataset): expanded_df["is_to_external_air"] == expanded_df["is_to_external_air_ending"] ) - ] + ] elif component == "roof": expanded_df = expanded_df[ (expanded_df["is_pitched"] == expanded_df["is_pitched_ending"]) @@ -549,7 +551,7 @@ class TrainingDataset(BaseDataset): expanded_df["has_dwelling_above"] == expanded_df["has_dwelling_above_ending"] ) - ] + ] return expanded_df @@ -695,10 +697,14 @@ class TrainingDataset(BaseDataset): cleaned_lookup_df_for_key = pd.DataFrame(cleaned_lookup[cleaned_key]) # We handle a specific edge case where we're missing information for the original description - descriptions = [x for x in self.df[left_on_starting].unique() if pd.notnull(x)] + descriptions = [ + x for x in self.df[left_on_starting].unique() if pd.notnull(x) + ] # take any not in the cleaned lookup missing_descriptions = [ - x for x in descriptions if x not in cleaned_lookup_df_for_key["original_description"].values + x + for x in descriptions + if x not in cleaned_lookup_df_for_key["original_description"].values ] if missing_descriptions: # We handle them here @@ -710,9 +716,12 @@ class TrainingDataset(BaseDataset): cleaned_data.append( { "original_description": x, - "clean_description": desc_cleaner.description.replace("(assumed)", - "").rstrip().capitalize(), - **cleaned + "clean_description": desc_cleaner.description.replace( + "(assumed)", "" + ) + .rstrip() + .capitalize(), + **cleaned, } ) cleaned_lookup_df_for_key = pd.concat( diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index e56faf7c..14c7f247 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -3,6 +3,7 @@ This script contains standard tables which are defined in rdsap. The most recent based on the 2012 version, however the government is currently working on releasing a new version, and there we will need to re-visit this """ + import pandas as pd age_band_data = [ @@ -11,84 +12,91 @@ age_band_data = [ "England_Wales": "before 1900", "Scotland": "before 1919", "Northern_Ireland": "before 1919", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "B", "England_Wales": "1900-1929", "Scotland": "1919-1929", "Northern_Ireland": "1919-1929", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "C", "England_Wales": "1930-1949", "Scotland": "1930-1949", "Northern_Ireland": "1930-1949", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "D", "England_Wales": "1950-1966", "Scotland": "1950-1964", "Northern_Ireland": "1950-1973", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "E", "England_Wales": "1967-1975", "Scotland": "1965-1975", "Northern_Ireland": "1974-1977", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "F", "England_Wales": "1976-1982", "Scotland": "1976-1983", "Northern_Ireland": "1978-1985", - "Park_home_UK": "before 1983" + "Park_home_UK": "before 1983", }, { "age_band": "G", "England_Wales": "1983-1990", "Scotland": "1984-1991", "Northern_Ireland": "1986-1991", - "Park_home_UK": "1983-1995" + "Park_home_UK": "1983-1995", }, { "age_band": "H", "England_Wales": "1991-1995", "Scotland": "1992-1998", "Northern_Ireland": "1992-1999", - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "I", "England_Wales": "1996-2002", "Scotland": "1999-2002", "Northern_Ireland": "2000-2006", - "Park_home_UK": "1996-2005" + "Park_home_UK": "1996-2005", }, { "age_band": "J", "England_Wales": "2003-2006", "Scotland": "2003-2007", "Northern_Ireland": None, - "Park_home_UK": None + "Park_home_UK": None, }, { "age_band": "K", "England_Wales": "2007-2011", "Scotland": "2008-2011", "Northern_Ireland": "2007-2013", - "Park_home_UK": "2006 onwards" + "Park_home_UK": "2006 onwards", }, { "age_band": "L", "England_Wales": "2012 onwards", "Scotland": "2012 onwards", "Northern_Ireland": "2014 onwards", - "Park_home_UK": None + "Park_home_UK": None, + }, + { + "age_band": "L", + "England_Wales": "2012-2021", + "Scotland": "2012-2023", + "Northern_Ireland": "2014-2022", + "Park_home_UK": None, }, ] @@ -102,32 +110,109 @@ england_wales_age_band_lookup = { ######################################################################################################################## default_wall_thickness = [ { - "type": "stone", "A": 500, "B": 500, "C": 500, "D": 500, "E": 450, "F": 420, "G": 420, "H": 420, - "I": 450, "J": 450, "K": 450, "L": 450 + "type": "stone", + "A": 500, + "B": 500, + "C": 500, + "D": 500, + "E": 450, + "F": 420, + "G": 420, + "H": 420, + "I": 450, + "J": 450, + "K": 450, + "L": 450, }, { - "type": "solid brick", "A": 220, "B": 220, "C": 220, "D": 220, "E": 240, "F": 250, "G": 270, "H": 270, - "I": 300, "J": 300, "K": 300, "L": 300 + "type": "solid brick", + "A": 220, + "B": 220, + "C": 220, + "D": 220, + "E": 240, + "F": 250, + "G": 270, + "H": 270, + "I": 300, + "J": 300, + "K": 300, + "L": 300, }, { - "type": "cavity", "A": 250, "B": 250, "C": 250, "D": 250, "E": 250, "F": 260, "G": 270, "H": 270, - "I": 300, "J": 300, "K": 300, "L": 300 + "type": "cavity", + "A": 250, + "B": 250, + "C": 250, + "D": 250, + "E": 250, + "F": 260, + "G": 270, + "H": 270, + "I": 300, + "J": 300, + "K": 300, + "L": 300, }, { - "type": "timber frame", "A": 150, "B": 150, "C": 150, "D": 250, "E": 270, "F": 270, "G": 270, "H": 270, - "I": 300, "J": 300, "K": 300, "L": 300 + "type": "timber frame", + "A": 150, + "B": 150, + "C": 150, + "D": 250, + "E": 270, + "F": 270, + "G": 270, + "H": 270, + "I": 300, + "J": 300, + "K": 300, + "L": 300, }, { - "type": "cob", "A": 540, "B": 540, "C": 540, "D": 540, "E": 540, "F": 540, "G": 560, "H": 560, "I": 590, - "J": 590, "K": 590, "L": 590 + "type": "cob", + "A": 540, + "B": 540, + "C": 540, + "D": 540, + "E": 540, + "F": 540, + "G": 560, + "H": 560, + "I": 590, + "J": 590, + "K": 590, + "L": 590, }, { - "type": "system build", "A": 250, "B": 250, "C": 250, "D": 250, "E": 250, "F": 300, "G": 300, "H": 300, - "I": 300, "J": 300, "K": 300, "L": 300 + "type": "system build", + "A": 250, + "B": 250, + "C": 250, + "D": 250, + "E": 250, + "F": 300, + "G": 300, + "H": 300, + "I": 300, + "J": 300, + "K": 300, + "L": 300, }, { - "type": "park home", "A": None, "B": None, "C": None, "D": None, "E": None, "F": 50, "G": 50, - "H": None, "I": 75, "J": 100, "K": 100, "L": 100 + "type": "park home", + "A": None, + "B": None, + "C": None, + "D": None, + "E": None, + "F": 50, + "G": 50, + "H": None, + "I": 75, + "J": 100, + "K": 100, + "L": 100, }, ] @@ -170,33 +255,384 @@ wall_types = [ u_values = [ ["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], ["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["1.7", "1.7", "1.7", "1.7", "1.7", "1.0", "0.60", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["0.55", "0.55", "0.55", "0.55", "0.55", "0.45", "0.35", "0.35", "0.30", "0.25", "0.21", "0.21"], - ["0.32", "0.32", "0.32", "0.32", "0.32", "0.28", "0.24", "0.24", "0.21", "0.19", "0.17", "0.16"], - ["0.23", "0.23", "0.23", "0.23", "0.23", "0.21", "0.18", "0.18", "0.17", "0.15", "0.14", "0.14"], - ["0.18", "0.18", "0.18", "0.18", "0.18", "0.17", "0.15", "0.15", "0.14", "0.13", "0.12", "0.12"], - ["0.80", "0.80", "0.80", "0.80", "0.80", "0.80", "0.60", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["0.40", "0.40", "0.40", "0.40", "0.40", "0.40", "0.35", "0.35", "0.30", "0.25", "0.21", "0.21"], - ["0.26", "0.26", "0.26", "0.26", "0.26", "0.26", "0.24", "0.24", "0.21", "0.19", "0.17", "0.16"], - ["0.20", "0.20", "0.20", "0.20", "0.20", "0.20", "0.18", "0.18", "0.17", "0.15", "0.14", "0.14"], - ["0.16", "0.16", "0.16", "0.16", "0.16", "0.16", "0.15", "0.15", "0.14", "0.13", "0.12", "0.12"], - ["1.5", "1.5", "1.5", "1.5", "1.5", "1.0", "0.60", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["0.53", "0.53", "0.53", "0.53", "0.53", "0.45", "0.35", "0.35", "0.30", "0.25", "0.21", "0.21"], - ["0.32", "0.32", "0.32", "0.32", "0.32", "0.30", "0.24", "0.24", "0.21", "0.19", "0.17", "0.16"], - ["0.23", "0.23", "0.23", "0.23", "0.23", "0.21", "0.18", "0.18", "0.17", "0.15", "0.14", "0.14"], - ["0.18", "0.18", "0.18", "0.18", "0.18", "0.17", "0.15", "0.15", "0.14", "0.13", "0.12", "0.12"], - ["0.7", "0.7", "0.7", "0.7", "0.7", "0.40", "0.35", "0.35", "0.45", "0.35", "0.30", "0.28"], - ["0.37", "0.37", "0.37", "0.37", "0.37", "0.27", "0.25", "0.25", "0.25", "0.25", "0.21", "0.21"], - ["0.25", "0.25", "0.25", "0.25", "0.25", "0.20", "0.19", "0.19", "0.19", "0.19", "0.17", "0.16"], - ["0.19", "0.19", "0.19", "0.19", "0.19", "0.16", "0.15", "0.15", "0.15", "0.15", "0.14", "0.14"], - ["0.16", "0.16", "0.16", "0.16", "0.16", "0.13", "0.13", "0.13", "0.13", "0.13", "0.12", "0.12"], - ["2.5", "1.9", "1.9", "1.0", "0.80", "0.45", "0.40", "0.40", "0.40", "0.35", "0.30", "0.28"], - ["0.60", "0.55", "0.55", "0.40", "0.40", "0.40", "0.40", "0.40", "0.40", "0.35", "0.30", "0.28"], - ["2.0", "2.0", "2.0", "2.0", "1.7", "1.0", "0.60", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["0.60", "0.60", "0.60", "0.60", "0.55", "0.45", "0.35", "0.35", "0.30", "0.25", "0.21", "0.21"], - ["0.35", "0.35", "0.35", "0.35", "0.35", "0.32", "0.24", "0.24", "0.21", "0.19", "0.17", "0.16"], - ["0.25", "0.25", "0.25", "0.25", "0.25", "0.21", "0.18", "0.18", "0.17", "0.15", "0.14", "0.14"], - ["0.18", "0.18", "0.18", "0.18", "0.18", "0.17", "0.15", "0.15", "0.14", "0.13", "0.12", "0.12"], + [ + "1.7", + "1.7", + "1.7", + "1.7", + "1.7", + "1.0", + "0.60", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + ], + [ + "0.55", + "0.55", + "0.55", + "0.55", + "0.55", + "0.45", + "0.35", + "0.35", + "0.30", + "0.25", + "0.21", + "0.21", + ], + [ + "0.32", + "0.32", + "0.32", + "0.32", + "0.32", + "0.28", + "0.24", + "0.24", + "0.21", + "0.19", + "0.17", + "0.16", + ], + [ + "0.23", + "0.23", + "0.23", + "0.23", + "0.23", + "0.21", + "0.18", + "0.18", + "0.17", + "0.15", + "0.14", + "0.14", + ], + [ + "0.18", + "0.18", + "0.18", + "0.18", + "0.18", + "0.17", + "0.15", + "0.15", + "0.14", + "0.13", + "0.12", + "0.12", + ], + [ + "0.80", + "0.80", + "0.80", + "0.80", + "0.80", + "0.80", + "0.60", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + ], + [ + "0.40", + "0.40", + "0.40", + "0.40", + "0.40", + "0.40", + "0.35", + "0.35", + "0.30", + "0.25", + "0.21", + "0.21", + ], + [ + "0.26", + "0.26", + "0.26", + "0.26", + "0.26", + "0.26", + "0.24", + "0.24", + "0.21", + "0.19", + "0.17", + "0.16", + ], + [ + "0.20", + "0.20", + "0.20", + "0.20", + "0.20", + "0.20", + "0.18", + "0.18", + "0.17", + "0.15", + "0.14", + "0.14", + ], + [ + "0.16", + "0.16", + "0.16", + "0.16", + "0.16", + "0.16", + "0.15", + "0.15", + "0.14", + "0.13", + "0.12", + "0.12", + ], + [ + "1.5", + "1.5", + "1.5", + "1.5", + "1.5", + "1.0", + "0.60", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + ], + [ + "0.53", + "0.53", + "0.53", + "0.53", + "0.53", + "0.45", + "0.35", + "0.35", + "0.30", + "0.25", + "0.21", + "0.21", + ], + [ + "0.32", + "0.32", + "0.32", + "0.32", + "0.32", + "0.30", + "0.24", + "0.24", + "0.21", + "0.19", + "0.17", + "0.16", + ], + [ + "0.23", + "0.23", + "0.23", + "0.23", + "0.23", + "0.21", + "0.18", + "0.18", + "0.17", + "0.15", + "0.14", + "0.14", + ], + [ + "0.18", + "0.18", + "0.18", + "0.18", + "0.18", + "0.17", + "0.15", + "0.15", + "0.14", + "0.13", + "0.12", + "0.12", + ], + [ + "0.7", + "0.7", + "0.7", + "0.7", + "0.7", + "0.40", + "0.35", + "0.35", + "0.45", + "0.35", + "0.30", + "0.28", + ], + [ + "0.37", + "0.37", + "0.37", + "0.37", + "0.37", + "0.27", + "0.25", + "0.25", + "0.25", + "0.25", + "0.21", + "0.21", + ], + [ + "0.25", + "0.25", + "0.25", + "0.25", + "0.25", + "0.20", + "0.19", + "0.19", + "0.19", + "0.19", + "0.17", + "0.16", + ], + [ + "0.19", + "0.19", + "0.19", + "0.19", + "0.19", + "0.16", + "0.15", + "0.15", + "0.15", + "0.15", + "0.14", + "0.14", + ], + [ + "0.16", + "0.16", + "0.16", + "0.16", + "0.16", + "0.13", + "0.13", + "0.13", + "0.13", + "0.13", + "0.12", + "0.12", + ], + [ + "2.5", + "1.9", + "1.9", + "1.0", + "0.80", + "0.45", + "0.40", + "0.40", + "0.40", + "0.35", + "0.30", + "0.28", + ], + [ + "0.60", + "0.55", + "0.55", + "0.40", + "0.40", + "0.40", + "0.40", + "0.40", + "0.40", + "0.35", + "0.30", + "0.28", + ], + [ + "2.0", + "2.0", + "2.0", + "2.0", + "1.7", + "1.0", + "0.60", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + ], + [ + "0.60", + "0.60", + "0.60", + "0.60", + "0.55", + "0.45", + "0.35", + "0.35", + "0.30", + "0.25", + "0.21", + "0.21", + ], + [ + "0.35", + "0.35", + "0.35", + "0.35", + "0.35", + "0.32", + "0.24", + "0.24", + "0.21", + "0.19", + "0.17", + "0.16", + ], + [ + "0.25", + "0.25", + "0.25", + "0.25", + "0.25", + "0.21", + "0.18", + "0.18", + "0.17", + "0.15", + "0.14", + "0.14", + ], + [ + "0.18", + "0.18", + "0.18", + "0.18", + "0.18", + "0.17", + "0.15", + "0.15", + "0.14", + "0.13", + "0.12", + "0.12", + ], ] age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"] @@ -210,8 +646,13 @@ for i, wall_type in enumerate(wall_types): parkhome_wall_uvalues = [ {"Wall_type": "Park home as built", "F": "1.7", "G": "1.2", "I": "0.7", "K": "0.6"}, - {"Wall_type": "Park home with additional insulation", "F": "s1.1.2", "G": "s1.1.2", "I": "s1.1.2", - "K": "s1.1.2"} + { + "Wall_type": "Park home with additional insulation", + "F": "s1.1.2", + "G": "s1.1.2", + "I": "s1.1.2", + "K": "s1.1.2", + }, ] wall_uvalues.extend(parkhome_wall_uvalues) @@ -229,16 +670,12 @@ epc_wall_description_map = { "Cavity wall, as built, insulated": "Filled cavity", "Cavity wall, with external insulation": "Unfilled cavity with 100 mm external or internal insulation", "Cavity wall, insulated": "Filled cavity", - 'Cavity wall, partial insulation': "Filled cavity", - + "Cavity wall, partial insulation": "Filled cavity", "Cavity wall,": "Cavity as built", # General case of cavity wall without further details - "Cavity wall, filled cavity and external insulation": - "Filled cavity with 100 mm external or internal insulation", - "Cavity wall, filled cavity and internal insulation": - "Filled cavity with 100 mm external or internal insulation", + "Cavity wall, filled cavity and external insulation": "Filled cavity with 100 mm external or internal insulation", + "Cavity wall, filled cavity and internal insulation": "Filled cavity with 100 mm external or internal insulation", "Cavity wall, with internal insulation": "Unfilled cavity with 100 mm external or internal insulation", "Cavity wall, no insulation": "Cavity as built", - ############################ # Solid brick wall mappings ############################ @@ -247,7 +684,6 @@ epc_wall_description_map = { "Solid brick, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation", "Solid brick, with external insulation": "Stone/solid brick with 100 mm external or internal insulation", "Solid brick, as built, partial insulation": "Stone/solid brick with 50 mm external or internal insulation", - ############################ # Timber frame wall mappings ############################ @@ -262,33 +698,29 @@ epc_wall_description_map = { # Sandstone/limestones wall mappings ############################ "Sandstone or limestone, as built, no insulation": "Stone: sandstone or limestone as built", - "Sandstone or limestone, with internal insulation": - "Stone/solid brick with 100 mm external or internal insulation", + "Sandstone or limestone, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation", "Sandstone or limestone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal " - "insulation", + "insulation", "Sandstone, as built, no insulation": "Stone: sandstone or limestone as built", - "Sandstone or limestone, as built, insulated": - "Stone/solid brick with 100 mm external or internal insulation", + "Sandstone or limestone, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation", "Sandstone, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation", "Sandstone, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation", "Sandstone or limestone, with external insulation": "Stone/solid brick with 100 mm external or internal " - "insulation", + "insulation", "Sandstone, with external insulation": "Stone/solid brick with 100 mm external or internal insulation", "Sandstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal insulation", - ############################ # Granite/whinstone wall mappings ############################ "Granite or whinstone, as built, no insulation": "Stone: granite or whinstone as built", "Granite or whinstone, with internal insulation": "Stone/solid brick with 100 mm external or internal " - "insulation", + "insulation", "Granite or whinstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal " - "insulation", + "insulation", "Granite or whinstone, as built, insulated": "Stone/solid brick with 100 mm external or internal " - "insulation", + "insulation", "Granite or whinstone, with external insulation": "Stone/solid brick with 100 mm external or internal " - "insulation", - + "insulation", ############################ # System built wall mappings ############################ @@ -297,15 +729,13 @@ epc_wall_description_map = { "System built, with internal insulation": "System build with 100 mm external or internal insulation", "System built, with external insulation": "System build with 100 mm external or internal insulation", "System built, as built, insulated": "System build with 100 mm external or internal insulation", - ############################ # Cob wall mappings ############################ "Cob, as built": "Cob as built", "Cob, with external insulation": "Cob with 100 mm external or internal insulation", "Cob, with internal insulation": "Cob with 100 mm external or internal insulation", - 'Cob,': "Cob as built", - + "Cob,": "Cob as built", ############################ # Park home mappings ############################ @@ -321,20 +751,71 @@ epc_wall_description_map = { ######################################################################################################################## s9_list = [ - {"Insulation_thickness_mm": None, "Slates_or_tiles_U_value_W_m2K": 2.3, "Thatched_roof_U_value_W_m2K": 0.35}, - {"Insulation_thickness_mm": 12, "Slates_or_tiles_U_value_W_m2K": 1.5, "Thatched_roof_U_value_W_m2K": 0.32}, - {"Insulation_thickness_mm": 25, "Slates_or_tiles_U_value_W_m2K": 1.0, "Thatched_roof_U_value_W_m2K": 0.30}, - {"Insulation_thickness_mm": 50, "Slates_or_tiles_U_value_W_m2K": 0.68, "Thatched_roof_U_value_W_m2K": 0.25}, - {"Insulation_thickness_mm": 75, "Slates_or_tiles_U_value_W_m2K": 0.50, "Thatched_roof_U_value_W_m2K": 0.22}, - {"Insulation_thickness_mm": 100, "Slates_or_tiles_U_value_W_m2K": 0.40, "Thatched_roof_U_value_W_m2K": 0.20}, - {"Insulation_thickness_mm": 150, "Slates_or_tiles_U_value_W_m2K": 0.30, "Thatched_roof_U_value_W_m2K": 0.17}, - {"Insulation_thickness_mm": 200, "Slates_or_tiles_U_value_W_m2K": 0.21, "Thatched_roof_U_value_W_m2K": 0.14}, - {"Insulation_thickness_mm": 250, "Slates_or_tiles_U_value_W_m2K": 0.17, "Thatched_roof_U_value_W_m2K": 0.12}, - {"Insulation_thickness_mm": 270, "Slates_or_tiles_U_value_W_m2K": 0.16, "Thatched_roof_U_value_W_m2K": 0.12}, - {"Insulation_thickness_mm": 300, "Slates_or_tiles_U_value_W_m2K": 0.14, "Thatched_roof_U_value_W_m2K": 0.11}, - {"Insulation_thickness_mm": 350, "Slates_or_tiles_U_value_W_m2K": 0.12, "Thatched_roof_U_value_W_m2K": 0.10}, - {"Insulation_thickness_mm": 400, "Slates_or_tiles_U_value_W_m2K": 0.11, - "Thatched_roof_U_value_W_m2K": 0.09}, + { + "Insulation_thickness_mm": None, + "Slates_or_tiles_U_value_W_m2K": 2.3, + "Thatched_roof_U_value_W_m2K": 0.35, + }, + { + "Insulation_thickness_mm": 12, + "Slates_or_tiles_U_value_W_m2K": 1.5, + "Thatched_roof_U_value_W_m2K": 0.32, + }, + { + "Insulation_thickness_mm": 25, + "Slates_or_tiles_U_value_W_m2K": 1.0, + "Thatched_roof_U_value_W_m2K": 0.30, + }, + { + "Insulation_thickness_mm": 50, + "Slates_or_tiles_U_value_W_m2K": 0.68, + "Thatched_roof_U_value_W_m2K": 0.25, + }, + { + "Insulation_thickness_mm": 75, + "Slates_or_tiles_U_value_W_m2K": 0.50, + "Thatched_roof_U_value_W_m2K": 0.22, + }, + { + "Insulation_thickness_mm": 100, + "Slates_or_tiles_U_value_W_m2K": 0.40, + "Thatched_roof_U_value_W_m2K": 0.20, + }, + { + "Insulation_thickness_mm": 150, + "Slates_or_tiles_U_value_W_m2K": 0.30, + "Thatched_roof_U_value_W_m2K": 0.17, + }, + { + "Insulation_thickness_mm": 200, + "Slates_or_tiles_U_value_W_m2K": 0.21, + "Thatched_roof_U_value_W_m2K": 0.14, + }, + { + "Insulation_thickness_mm": 250, + "Slates_or_tiles_U_value_W_m2K": 0.17, + "Thatched_roof_U_value_W_m2K": 0.12, + }, + { + "Insulation_thickness_mm": 270, + "Slates_or_tiles_U_value_W_m2K": 0.16, + "Thatched_roof_U_value_W_m2K": 0.12, + }, + { + "Insulation_thickness_mm": 300, + "Slates_or_tiles_U_value_W_m2K": 0.14, + "Thatched_roof_U_value_W_m2K": 0.11, + }, + { + "Insulation_thickness_mm": 350, + "Slates_or_tiles_U_value_W_m2K": 0.12, + "Thatched_roof_U_value_W_m2K": 0.10, + }, + { + "Insulation_thickness_mm": 400, + "Slates_or_tiles_U_value_W_m2K": 0.11, + "Thatched_roof_U_value_W_m2K": 0.09, + }, ] s10_list = [ @@ -347,7 +828,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 2.3, "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": None + "Park_home": None, }, { "Age_band": "E", @@ -358,7 +839,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 1.5, "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": None + "Park_home": None, }, { "Age_band": "F", @@ -369,7 +850,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.80, "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": 1.7 + "Park_home": 1.7, }, { "Age_band": "G", @@ -380,7 +861,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": "0.50", "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": 0.6 + "Park_home": 0.6, }, { "Age_band": "H", @@ -391,7 +872,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.35, "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": None + "Park_home": None, }, { "Age_band": "I", @@ -402,7 +883,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.35, "Thatched_roof": 0.35, "Thatched_roof_room_in_roof": 0.25, - "Park_home": 0.35 + "Park_home": 0.35, }, { "Age_band": "J", @@ -413,7 +894,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.30, "Thatched_roof": 0.30, "Thatched_roof_room_in_roof": 0.25, - "Park_home": None + "Park_home": None, }, { "Age_band": "K", @@ -424,7 +905,7 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.25, "Thatched_roof": 0.25, "Thatched_roof_room_in_roof": 0.25, - "Park_home": 0.30 + "Park_home": 0.30, }, { "Age_band": "L", @@ -435,8 +916,8 @@ s10_list = [ "Room_in_roof_slates_or_tiles": 0.18, "Thatched_roof": 0.18, "Thatched_roof_room_in_roof": 0.18, - "Park_home": None - } + "Park_home": None, + }, ] table_s9 = pd.DataFrame(s9_list) @@ -452,22 +933,70 @@ table_s10 = pd.DataFrame(s10_list) ######################################################################################################################## s11_list = [ - {"Age_band": "A, B", "Floor_construction": "suspended timber", "England_Wales": 0, "Scotland": 0, - "Northern_Ireland": 0, "Park_home": 0}, - {"Age_band": "C to F", "Floor_construction": "solid", "England_Wales": 0, "Scotland": 0, - "Northern_Ireland": 0, "Park_home": 0}, - {"Age_band": "G", "Floor_construction": "solid", "England_Wales": 0, "Scotland": 0, - "Northern_Ireland": 0, "Park_home": 25}, - {"Age_band": "H", "Floor_construction": "solid", "England_Wales": 0, "Scotland": 25, - "Northern_Ireland": 25, "Park_home": 0}, - {"Age_band": "I", "Floor_construction": "solid", "England_Wales": 25, "Scotland": 50, - "Northern_Ireland": 50, "Park_home": 50}, - {"Age_band": "J", "Floor_construction": "solid", "England_Wales": 75, "Scotland": 75, - "Northern_Ireland": 0, "Park_home": 0}, - {"Age_band": "K", "Floor_construction": "solid", "England_Wales": 100, "Scotland": 100, - "Northern_Ireland": 100, "Park_home": 70}, - {"Age_band": "L", "Floor_construction": "solid", "England_Wales": 100, "Scotland": 120, - "Northern_Ireland": 100, "Park_home": 0}, + { + "Age_band": "A, B", + "Floor_construction": "suspended timber", + "England_Wales": 0, + "Scotland": 0, + "Northern_Ireland": 0, + "Park_home": 0, + }, + { + "Age_band": "C to F", + "Floor_construction": "solid", + "England_Wales": 0, + "Scotland": 0, + "Northern_Ireland": 0, + "Park_home": 0, + }, + { + "Age_band": "G", + "Floor_construction": "solid", + "England_Wales": 0, + "Scotland": 0, + "Northern_Ireland": 0, + "Park_home": 25, + }, + { + "Age_band": "H", + "Floor_construction": "solid", + "England_Wales": 0, + "Scotland": 25, + "Northern_Ireland": 25, + "Park_home": 0, + }, + { + "Age_band": "I", + "Floor_construction": "solid", + "England_Wales": 25, + "Scotland": 50, + "Northern_Ireland": 50, + "Park_home": 50, + }, + { + "Age_band": "J", + "Floor_construction": "solid", + "England_Wales": 75, + "Scotland": 75, + "Northern_Ireland": 0, + "Park_home": 0, + }, + { + "Age_band": "K", + "Floor_construction": "solid", + "England_Wales": 100, + "Scotland": 100, + "Northern_Ireland": 100, + "Park_home": 70, + }, + { + "Age_band": "L", + "Floor_construction": "solid", + "England_Wales": 100, + "Scotland": 120, + "Northern_Ireland": 100, + "Park_home": 0, + }, ] table_s11 = pd.DataFrame(s11_list) @@ -481,21 +1010,90 @@ table_s11 = pd.DataFrame(s11_list) ######################################################################################################################## s12_list = [ - {"age_band": "A", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "B", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "C", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "D", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "E", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "F", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "G", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - - {"age_band": "H", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - {"age_band": "I", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, - - {"age_band": "J", "insulation_0": 0.25, "insulation_50": 0.25, "insulation_100": 0.25, "insulation_150": 0.22}, - - {"age_band": "K", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22}, - {"age_band": "L", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22}, + { + "age_band": "A", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "B", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "C", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "D", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "E", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "F", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "G", + "insulation_0": 1.2, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "H", + "insulation_0": 0.51, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "I", + "insulation_0": 0.51, + "insulation_50": 0.5, + "insulation_100": 0.3, + "insulation_150": 0.22, + }, + { + "age_band": "J", + "insulation_0": 0.25, + "insulation_50": 0.25, + "insulation_100": 0.25, + "insulation_150": 0.22, + }, + { + "age_band": "K", + "insulation_0": 0.22, + "insulation_50": 0.22, + "insulation_100": 0.22, + "insulation_150": 0.22, + }, + { + "age_band": "L", + "insulation_0": 0.22, + "insulation_50": 0.22, + "insulation_100": 0.22, + "insulation_150": 0.22, + }, ] table_s12 = pd.DataFrame(s12_list)