From 92fcbe8cdb86b58be282cb0c97ae8bb183307954 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sat, 1 Nov 2025 15:34:45 +0000 Subject: [PATCH] amend etl code for new october data --- etl/epc/DataProcessor.py | 11 +- etl/epc/Pipeline.py | 10 ++ etl/epc/property_change_app.py | 2 +- .../epc_attributes/FloorAttributes.py | 59 ++++--- .../epc_attributes/RoofAttributes.py | 89 ++++++++--- .../epc_attributes/WallAttributes.py | 146 ++++++++++-------- recommendations/rdsap_tables.py | 102 ++++++++++-- recommendations/recommendation_utils.py | 4 +- 8 files changed, 295 insertions(+), 128 deletions(-) diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 41dca943..682e9e78 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -21,7 +21,7 @@ from etl.epc.settings import ( ENDING_SUFFIX_COMPONENT_COLS, POTENTIAL_COLUMNS, EFFICIENCY_FEATURES, - DATA_ANOMALY_MATCHES + DATA_ANOMALY_MATCHES, ) from recommendations.rdsap_tables import FLOOR_LEVEL_MAP @@ -249,7 +249,8 @@ class EPCDataProcessor: # Map all anomaly values to None data_anomaly_map = dict( zip( - DATA_ANOMALY_MATCHES, [None] * len(DATA_ANOMALY_MATCHES), + DATA_ANOMALY_MATCHES, + [None] * len(DATA_ANOMALY_MATCHES), ) ) @@ -749,6 +750,12 @@ class EPCDataProcessor: self.data = self.data[~pd.isnull(self.data["HOTWATER_DESCRIPTION"])] self.data = self.data[~pd.isnull(self.data["ROOF_DESCRIPTION"])] + # Remove any walls described as Basement walls since these are non-standard + # TODO: CHECK IF WE SHOULD MAP THESE U VALUES INSTEAD + index_to_remove = self.data["WALLS_DESCRIPTION"] == "Basement wall" + print(f"Removing {index_to_remove.sum()} records with basement walls") + self.data = self.data[~index_to_remove] + # Because park homes are surveyed unusually (for example, we don't have u-values to # look up for their different components, they need to be collected in survey and aren't reflected in # EPCs) we'll ignore them from the model diff --git a/etl/epc/Pipeline.py b/etl/epc/Pipeline.py index c03abfcf..9f427c59 100644 --- a/etl/epc/Pipeline.py +++ b/etl/epc/Pipeline.py @@ -112,6 +112,16 @@ clean_lookup["mainheatcont-description"] = new_mainheatcont_mapping.to_dict( orient="records" ) +# TEMP FIX - GRANITE OR WHINSTONE BOOLEAN ISSUE +new_walls_description_mapping = pd.DataFrame(clean_lookup["walls-description"]) +new_walls_description_mapping.loc[ + new_walls_description_mapping["original_description"].str.contains("Granite"), + "is_granite_or_whinstone", +] = True +clean_lookup["walls-description"] = new_walls_description_mapping.to_dict( + orient="records" +) + class EPCPipeline: """ diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index c985567d..cdb7cfb8 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -12,7 +12,7 @@ def main(): """ directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()] - # directories = directories[0:3] + # directories = directories[235:275] epc_pipeline = EPCPipeline( directories=directories, diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 6def93f0..23c7dd8e 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -1,17 +1,26 @@ import re from typing import Dict, Union from BaseUtility import Definitions -from etl.epc_clean.epc_attributes.attribute_utils import extract_thermal_transmittance, extract_component_types +from etl.epc_clean.epc_attributes.attribute_utils import ( + extract_thermal_transmittance, + extract_component_types, +) class FloorAttributes(Definitions): DWELLING_BELOW = ["another dwelling below", "other premises below"] - FLOOR_TYPES = ["assumed", "to unheated space", "to external air", "suspended", "solid"] + FLOOR_TYPES = [ + "assumed", + "to unheated space", + "to external air", + "suspended", + "solid", + ] # For the short term, while we are still exploring the data, we maintain a list of error cases which # we want to ignore and consider as no data. - OBSERVED_ERRORS = ["Conservatory", "insulated"] + OBSERVED_ERRORS = ["Conservatory", "insulated", "Basement"] WELSH_TEXT = { "(anheddiad arall islaw)": "(another dwelling below)", @@ -35,32 +44,40 @@ class FloorAttributes(Definitions): "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation", "igçör awyr y tu allan, wedigçöi inswleiddio (rhagdybiaeth)": "to external air, insulated (assumed)", - "crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)" + "crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)", } def __init__(self, description: str): self.description: str = description.lower() - self.nodata = (not description) or (description in self.DATA_ANOMALY_MATCHES) or ( - description in self.OBSERVED_ERRORS) or (self.description == "sap05:floor") + self.nodata = ( + (not description) + or (description in self.DATA_ANOMALY_MATCHES) + or (description in self.OBSERVED_ERRORS) + or (self.description == "sap05:floor") + ) # Try and perform a translation, incase it's in welsh self.translate_welsh_text() if not self.nodata and not any( - rt in self.description for rt in - self.FLOOR_TYPES + self.DWELLING_BELOW + ["average thermal transmittance"] + rt in self.description + for rt in self.FLOOR_TYPES + + self.DWELLING_BELOW + + ["average thermal transmittance"] ): - raise ValueError('Invalid description') + raise ValueError("Invalid description") def translate_welsh_text(self): uvalue_match = re.search( - r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k', self.description + r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", + self.description, ) uvalue_match2 = re.search( - r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description + r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k", + self.description, ) # Step 2: Generalized translation with placeholder @@ -69,7 +86,7 @@ class FloorAttributes(Definitions): uvalue = uvalue_match.group(1) else: uvalue = uvalue_match2.group(1) - self.description = f'average thermal transmittance {uvalue} w/m-¦k' + self.description = f"average thermal transmittance {uvalue} w/m-¦k" else: translation = self.WELSH_TEXT.get(self.description) @@ -89,11 +106,15 @@ class FloorAttributes(Definitions): result, description = extract_thermal_transmittance(result, description) # floor type - result, description = extract_component_types(result, description, list_of_components=self.FLOOR_TYPES) + result, description = extract_component_types( + result, description, list_of_components=self.FLOOR_TYPES + ) # check if there is another dwelling below - result['another_property_below'] = "(another dwelling below)" in description or "(other premises below)" in \ - description + result["another_property_below"] = ( + "(another dwelling below)" in description + or "(other premises below)" in description + ) thickness_map = { "external insulation": "average", @@ -102,17 +123,17 @@ class FloorAttributes(Definitions): "partial insulation": "below average", "no insulation": "none", "additional insulation": "above average", - "insulated": "average" + "insulated": "average", } for key, value in thickness_map.items(): if key in description: - result['insulation_thickness'] = value + result["insulation_thickness"] = value break else: - result['insulation_thickness'] = None + result["insulation_thickness"] = None if result["another_property_below"]: result["thermal_transmittance"] = 0 - result["thermal_transmittance_unit"] = 'w/m-¦k' + result["thermal_transmittance_unit"] = "w/m-¦k" return result diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 2eacc951..153fb548 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -1,12 +1,28 @@ import re from typing import Dict, Union from BaseUtility import Definitions -from etl.epc_clean.epc_attributes.attribute_utils import extract_component_types, extract_thermal_transmittance +from etl.epc_clean.epc_attributes.attribute_utils import ( + extract_component_types, + extract_thermal_transmittance, +) class RoofAttributes(Definitions): - ROOF_TYPES = ['pitched', 'roof room', 'loft', 'flat', 'thatched', 'at rafters', 'assumed'] - DWELLING_ABOVE = ["another dwelling above", "other premises above", "other dwelling above"] + ROOF_TYPES = [ + "pitched", + "roof room", + "loft", + "flat", + "thatched", + "at rafters", + "assumed", + ] + DWELLING_ABOVE = [ + "another dwelling above", + "other premises above", + "other dwelling above", + "(same dwelling above)", + ] WELSH_TEXT = { "ar oleddf, dim inswleiddio": "pitched, no insulation", @@ -18,10 +34,10 @@ class RoofAttributes(Definitions): "ar oleddf, wedi?i inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", - "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', - "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', - "ar oleddf, wedi?i inswleiddio wrth y trawstia": 'pitched, insulated at rafters', - "ar oleddf, wedigçöi inswleiddio wrth y trawstia": 'pitched, insulated at rafters', + "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": "pitched, insulated at rafters", + "ar oleddf, wedi?i inswleiddio wrth y trawstiau": "pitched, insulated at rafters", + "ar oleddf, wedi?i inswleiddio wrth y trawstia": "pitched, insulated at rafters", + "ar oleddf, wedigçöi inswleiddio wrth y trawstia": "pitched, insulated at rafters", "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", @@ -43,9 +59,18 @@ class RoofAttributes(Definitions): } DEFAULT_KEYS = [ - 'thermal_transmittance', 'thermal_transmittance_unit', 'is_pitched', 'is_roof_room', - 'is_loft', 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed', 'has_dwelling_above', - 'is_valid', 'insulation_thickness' + "thermal_transmittance", + "thermal_transmittance_unit", + "is_pitched", + "is_roof_room", + "is_loft", + "is_flat", + "is_thatched", + "is_at_rafters", + "is_assumed", + "has_dwelling_above", + "is_valid", + "insulation_thickness", ] def __init__(self, description: str): @@ -54,14 +79,21 @@ class RoofAttributes(Definitions): """ self.description: str = description.lower().strip() - self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or self.description == "sap05:roof" + self.nodata = ( + not description + or description in self.DATA_ANOMALY_MATCHES + or self.description == "sap05:roof" + ) self.welsh_translation_search() if not self.nodata and not any( - rt in self.description for rt in self.ROOF_TYPES + self.DWELLING_ABOVE + ["average thermal transmittance"] + rt in self.description + for rt in self.ROOF_TYPES + + self.DWELLING_ABOVE + + ["average thermal transmittance"] ): - raise ValueError('Invalid description') + raise ValueError("Invalid description") def welsh_translation_search(self): """ @@ -76,7 +108,7 @@ class RoofAttributes(Definitions): r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", r"ar oleddf, (\d+mm) o inswleiddio yn y llofft", - r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft" + r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft", ] li_thickness_match = None for regex in loft_insulation_regexes: @@ -84,9 +116,14 @@ class RoofAttributes(Definitions): if li_thickness_match: break - uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description) + uvalue_search = re.search( + r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", + self.description, + ) uvalue_search2 = re.search( - r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description, re.IGNORECASE + r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k", + self.description, + re.IGNORECASE, ) # Step 2: Generalized translation with placeholder @@ -121,9 +158,13 @@ class RoofAttributes(Definitions): result, description = extract_thermal_transmittance(result, description) # roof type - result, description = extract_component_types(result, description, list_of_components=self.ROOF_TYPES) + result, description = extract_component_types( + result, description, list_of_components=self.ROOF_TYPES + ) - result["has_dwelling_above"] = any([x in description for x in self.DWELLING_ABOVE]) + result["has_dwelling_above"] = any( + [x in description for x in self.DWELLING_ABOVE] + ) for dwelling_above in self.DWELLING_ABOVE: description = description.replace(dwelling_above, "") @@ -136,7 +177,7 @@ class RoofAttributes(Definitions): # Search for a regular expression that matches 150 insulation match = re.search(r"(\d+\+?)\s*insulation", description) if match: - result['insulation_thickness'] = match.group(1) + result["insulation_thickness"] = match.group(1) # insulation thickness thickness_map = { @@ -149,21 +190,21 @@ class RoofAttributes(Definitions): } for key, value in thickness_map.items(): if key in description: - result['insulation_thickness'] = value + result["insulation_thickness"] = value # Remove the match from the description # description = description.replace(key, "") break # Extract insulation thickness in mm, if present - match = re.search(r'(\d+\+?)\s*mm', description) + match = re.search(r"(\d+\+?)\s*mm", description) if match: - result['insulation_thickness'] = match.group(1) + result["insulation_thickness"] = match.group(1) if "insulation_thickness" not in result: - result['insulation_thickness'] = None + result["insulation_thickness"] = None if result["has_dwelling_above"]: result["thermal_transmittance"] = 0 - result["thermal_transmittance_unit"] = 'w/m-¦k' + result["thermal_transmittance_unit"] = "w/m-¦k" return result diff --git a/etl/epc_clean/epc_attributes/WallAttributes.py b/etl/epc_clean/epc_attributes/WallAttributes.py index 49252552..8cf32a0b 100644 --- a/etl/epc_clean/epc_attributes/WallAttributes.py +++ b/etl/epc_clean/epc_attributes/WallAttributes.py @@ -3,76 +3,78 @@ from typing import Dict, Union from BaseUtility import Definitions from etl.epc_clean.epc_attributes.attribute_utils import ( extract_component_types, - extract_thermal_transmittance + extract_thermal_transmittance, ) class WallAttributes(Definitions): - WALL_TYPES = ['cavity wall', 'filled cavity', 'solid brick', 'system built', 'timber frame', 'granite or whinstone', - 'as built', 'cob', 'assumed', 'sandstone or limestone', "park home"] + WALL_TYPES = [ + "cavity wall", + "filled cavity", + "solid brick", + "system built", + "timber frame", + "granite or whinstone", + "as built", + "cob", + "assumed", + "sandstone or limestone", + "park home", + ] WELSH_TEXT = { - "Briciau solet, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": - "Solid brick, as built, no insulation (assumed)", - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)': - 'Cavity wall, as built, partial insulation (assumed)', - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol': - 'Cavity wall, as built, partial insulation', - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': - 'Cavity wall, as built, no insulation (assumed)', - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio': - 'Cavity wall, as built, no insulation', - 'Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': - 'Sandstone or limestone, as built, no insulation (assumed)', - 'Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio': - 'Sandstone or limestone, as built, no insulation', - 'Waliau ceudod, ceudod wediGÇÖi lenwi': 'Cavity wall, filled cavity', - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)': - 'Cavity wall, as built, insulated (assumed)', - 'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio': - 'Cavity wall, as built, insulated', - 'Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': - 'Granite or whinstone, as built, no insulation (assumed)', - 'Waliau ceudod,': 'Cavity wall, as built, no insulation', - 'Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)': - 'Timber frame, as built, insulated (assumed)', - 'Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio': - 'Timber frame, as built, insulated', - 'Gwenithfaen neu risgraig, gydag inswleiddio allanol': 'Granite or whinstone, with external insulation', - 'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': - 'System built, as built, no insulation (assumed)', - 'Tywodfaen, gydag inswleiddio mewnol': 'Sandstone or limestone, with internal insulation', - 'Waliau ceudod, ynysydd allanol a llenwi ceudod': 'Cavity wall, filled cavity and external insulation', - 'Gwenithfaen neu risgraig, gydag inswleiddio mewnol': 'Granite or whinstone, with internal insulation', - 'Ffr+óm bren, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)': - 'Timber frame, as built, partial insulation (assumed)', - 'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)': - 'System built, as built, insulated (assumed)', - 'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio': - 'System built, as built, insulated', - 'WediGÇÖu hadeiladu yn +¦l system, gydag inswleiddio allanol': 'System built, with external insulation', - 'Briciau solet, gydag inswleiddio mewnol': 'Solid brick, with internal insulation', - 'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)': - 'System built, as built, partial insulation (assumed)', - 'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol': - 'System built, as built, partial insulation', - 'Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': - 'Timber frame, as built, no insulation (assumed)', - 'Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio': - 'Timber frame, as built, no insulation', - 'Tywodfaen, gydag inswleiddio allanol': 'Sandstone or limestone, with external insulation', - 'Waliau ceudod, gydag inswleiddio allanol': 'Cavity wall, with external insulation', - 'Briciau solet, gydag inswleiddio allanol': 'Solid brick, with external insulation', + "Briciau solet, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Solid brick, as built, no insulation (assumed)", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "Cavity wall, as built, partial insulation (assumed)", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol": "Cavity wall, as built, partial insulation", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Cavity wall, as built, no insulation (assumed)", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Cavity wall, as built, no insulation", + "Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Sandstone or limestone, as built, no insulation (assumed)", + "Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Sandstone or limestone, as built, no insulation", + "Waliau ceudod, ceudod wediGÇÖi lenwi": "Cavity wall, filled cavity", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, insulated (assumed)", + "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Cavity wall, as built, insulated", + "Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Granite or whinstone, as built, no insulation (assumed)", + "Waliau ceudod,": "Cavity wall, as built, no insulation", + "Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Timber frame, as built, insulated (assumed)", + "Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Timber frame, as built, insulated", + "Gwenithfaen neu risgraig, gydag inswleiddio allanol": "Granite or whinstone, with external insulation", + "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "System built, as built, no insulation (assumed)", + "Tywodfaen, gydag inswleiddio mewnol": "Sandstone or limestone, with internal insulation", + "Waliau ceudod, ynysydd allanol a llenwi ceudod": "Cavity wall, filled cavity and external insulation", + "Gwenithfaen neu risgraig, gydag inswleiddio mewnol": "Granite or whinstone, with internal insulation", + "Ffr+óm bren, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "Timber frame, as built, partial insulation (assumed)", + "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "System built, as built, insulated (assumed)", + "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "System built, as built, insulated", + "WediGÇÖu hadeiladu yn +¦l system, gydag inswleiddio allanol": "System built, with external insulation", + "Briciau solet, gydag inswleiddio mewnol": "Solid brick, with internal insulation", + "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "System built, as built, partial insulation (assumed)", + "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol": "System built, as built, partial insulation", + "Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Timber frame, as built, no insulation (assumed)", + "Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Timber frame, as built, no insulation", + "Tywodfaen, gydag inswleiddio allanol": "Sandstone or limestone, with external insulation", + "Waliau ceudod, gydag inswleiddio allanol": "Cavity wall, with external insulation", + "Briciau solet, gydag inswleiddio allanol": "Solid brick, with external insulation", # Add in some corrections: - 'Co with external insulation': 'Cob, with external insulation', - 'Cowith external insulation': 'Cob, with external insulation', + "Co with external insulation": "Cob, with external insulation", + "Cowith external insulation": "Cob, with external insulation", } DEFAULT_KEYS = [ - 'thermal_transmittance', 'thermal_transmittance_unit', 'is_cavity_wall', 'is_filled_cavity', - 'is_solid_brick', 'is_system_built', 'is_timber_frame', 'is_granite_or_whinstone', - 'is_as_built', 'is_cob', 'is_assumed', 'is_sandstone_or_limestone', - 'insulation_thickness', 'external_insulation', 'internal_insulation' + "thermal_transmittance", + "thermal_transmittance_unit", + "is_cavity_wall", + "is_filled_cavity", + "is_solid_brick", + "is_system_built", + "is_timber_frame", + "is_granite_or_whinstone", + "is_as_built", + "is_cob", + "is_assumed", + "is_sandstone_or_limestone", + "insulation_thickness", + "external_insulation", + "internal_insulation", ] CORRECTIONS = { @@ -98,7 +100,9 @@ class WallAttributes(Definitions): :return: """ - uvalue_search = re.search(r"Trawsyriannedd thermol cyfartalog (\d+\.?\d*)", self.description) + uvalue_search = re.search( + r"Trawsyriannedd thermol cyfartalog (\d+\.?\d*)", self.description + ) if uvalue_search: uvalue = uvalue_search.group(1) @@ -123,7 +127,9 @@ class WallAttributes(Definitions): result, description = extract_thermal_transmittance(result, description) # wall type - result, description = extract_component_types(result, description, list_of_components=self.WALL_TYPES) + result, description = extract_component_types( + result, description, list_of_components=self.WALL_TYPES + ) # Handle some edge cases if "sandstone" in description and not result["is_sandstone_or_limestone"]: @@ -137,18 +143,18 @@ class WallAttributes(Definitions): "partial insulation": "below average", "no insulation": "none", "additional insulation": "above average", - "insulated": "average" + "insulated": "average", } for key, value in thickness_map.items(): if key in description: - result['insulation_thickness'] = value + result["insulation_thickness"] = value break else: - result['insulation_thickness'] = None + result["insulation_thickness"] = None # insulation type - result['external_insulation'] = 'external insulation' in description - result['internal_insulation'] = 'internal insulation' in description + result["external_insulation"] = "external insulation" in description + result["internal_insulation"] = "internal insulation" in description if result["is_filled_cavity"]: # If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation @@ -159,7 +165,11 @@ class WallAttributes(Definitions): else: result["insulation_thickness"] = "average" - if result["is_cavity_wall"] & result["is_as_built"] & (result["insulation_thickness"] == "average"): + if ( + result["is_cavity_wall"] + & result["is_as_built"] + & (result["insulation_thickness"] == "average") + ): result["is_filled_cavity"] = True return result diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index 14c7f247..46e7d083 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -98,6 +98,13 @@ age_band_data = [ "Northern_Ireland": "2014-2022", "Park_home_UK": None, }, + { + "age_band": "M", + "England_Wales": "2022 onwards", + "Scotland": "2024 onwards", + "Northern_Ireland": "2023 onwards", + "Park_home_UK": None, + }, ] england_wales_age_band_lookup = { @@ -123,6 +130,7 @@ default_wall_thickness = [ "J": 450, "K": 450, "L": 450, + "M": 450, }, { "type": "solid brick", @@ -138,6 +146,7 @@ default_wall_thickness = [ "J": 300, "K": 300, "L": 300, + "M": 300, }, { "type": "cavity", @@ -153,6 +162,7 @@ default_wall_thickness = [ "J": 300, "K": 300, "L": 300, + "M": 300, }, { "type": "timber frame", @@ -168,6 +178,7 @@ default_wall_thickness = [ "J": 300, "K": 300, "L": 300, + "M": 300, }, { "type": "cob", @@ -183,6 +194,7 @@ default_wall_thickness = [ "J": 590, "K": 590, "L": 590, + "M": 590, }, { "type": "system build", @@ -198,6 +210,7 @@ default_wall_thickness = [ "J": 300, "K": 300, "L": 300, + "M": 300, }, { "type": "park home", @@ -213,6 +226,7 @@ default_wall_thickness = [ "J": 100, "K": 100, "L": 100, + "M": 100, }, ] @@ -253,8 +267,36 @@ wall_types = [ ] u_values = [ - ["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], - ["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], + [ + "a", + "a", + "a", + "a", + "1.7b", + "1.0", + "0.6", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + "0.26", + ], + [ + "a", + "a", + "a", + "a", + "1.7b", + "1.0", + "0.6", + "0.60", + "0.45", + "0.35", + "0.30", + "0.28", + "0.26", + ], [ "1.7", "1.7", @@ -268,6 +310,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.55", @@ -282,6 +325,7 @@ u_values = [ "0.25", "0.21", "0.21", + "0.20", ], [ "0.32", @@ -296,6 +340,7 @@ u_values = [ "0.19", "0.17", "0.16", + "0.15", ], [ "0.23", @@ -310,6 +355,7 @@ u_values = [ "0.15", "0.14", "0.14", + "0.13", ], [ "0.18", @@ -324,6 +370,7 @@ u_values = [ "0.13", "0.12", "0.12", + "0.11", ], [ "0.80", @@ -338,6 +385,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.40", @@ -352,6 +400,7 @@ u_values = [ "0.25", "0.21", "0.21", + "0.20", ], [ "0.26", @@ -366,6 +415,7 @@ u_values = [ "0.19", "0.17", "0.16", + "0.15", ], [ "0.20", @@ -380,6 +430,7 @@ u_values = [ "0.15", "0.14", "0.14", + "0.13", ], [ "0.16", @@ -394,6 +445,7 @@ u_values = [ "0.13", "0.12", "0.12", + "0.11", ], [ "1.5", @@ -408,6 +460,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.53", @@ -422,6 +475,7 @@ u_values = [ "0.25", "0.21", "0.21", + "0.20", ], [ "0.32", @@ -436,6 +490,7 @@ u_values = [ "0.19", "0.17", "0.16", + "0.15", ], [ "0.23", @@ -450,6 +505,7 @@ u_values = [ "0.15", "0.14", "0.14", + "0.13", ], [ "0.18", @@ -464,6 +520,7 @@ u_values = [ "0.13", "0.12", "0.12", + "0.11", ], [ "0.7", @@ -478,6 +535,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.37", @@ -492,6 +550,7 @@ u_values = [ "0.25", "0.21", "0.21", + "0.20", ], [ "0.25", @@ -506,6 +565,7 @@ u_values = [ "0.19", "0.17", "0.16", + "0.15", ], [ "0.19", @@ -520,6 +580,7 @@ u_values = [ "0.15", "0.14", "0.14", + "0.13", ], [ "0.16", @@ -534,6 +595,7 @@ u_values = [ "0.13", "0.12", "0.12", + "0.11", ], [ "2.5", @@ -548,6 +610,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.60", @@ -562,6 +625,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "2.0", @@ -576,6 +640,7 @@ u_values = [ "0.35", "0.30", "0.28", + "0.26", ], [ "0.60", @@ -590,6 +655,7 @@ u_values = [ "0.25", "0.21", "0.21", + "0.20", ], [ "0.35", @@ -604,6 +670,7 @@ u_values = [ "0.19", "0.17", "0.16", + "0.15", ], [ "0.25", @@ -618,6 +685,7 @@ u_values = [ "0.15", "0.14", "0.14", + "0.13", ], [ "0.18", @@ -632,10 +700,11 @@ u_values = [ "0.13", "0.12", "0.12", + "0.11", ], ] -age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"] +age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"] wall_uvalues = [] for i, wall_type in enumerate(wall_types): @@ -645,13 +714,23 @@ for i, wall_type in enumerate(wall_types): wall_uvalues.append(row) parkhome_wall_uvalues = [ - {"Wall_type": "Park home as built", "F": "1.7", "G": "1.2", "I": "0.7", "K": "0.6"}, + { + "Wall_type": "Park home as built", + "F": "1.7", + "G": "1.2", + "I": "0.7", + "K": "0.6", + "L": "0.6", + "M": "0.6", + }, { "Wall_type": "Park home with additional insulation", "F": "s1.1.2", "G": "s1.1.2", "I": "s1.1.2", "K": "s1.1.2", + "L": "s1.1.2", + "M": "s1.1.2", }, ] @@ -713,14 +792,13 @@ epc_wall_description_map = { # Granite/whinstone wall mappings ############################ "Granite or whinstone, as built, no insulation": "Stone: granite or whinstone as built", - "Granite or whinstone, with internal insulation": "Stone/solid brick with 100 mm external or internal " - "insulation", - "Granite or whinstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal " - "insulation", - "Granite or whinstone, as built, insulated": "Stone/solid brick with 100 mm external or internal " - "insulation", - "Granite or whinstone, with external insulation": "Stone/solid brick with 100 mm external or internal " - "insulation", + "Granite or whinstone, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation", + "Granite or whin, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation", + "Granite or whinstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal insulation", + "Granite or whinstone, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation", + "Granite or whin, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation", + "Granite or whinstone, with external insulation": "Stone/solid brick with 100 mm external or internal insulation", + "Granite or whin, with external insulation": "Stone/solid brick with 100 mm external or internal insulation", ############################ # System built wall mappings ############################ diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 602684cf..7c39668a 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -205,7 +205,7 @@ def get_wall_u_value( mapped_value = wall_uvalues_df[ wall_uvalues_df["Wall_type"] == mapped_description - ][age_band].values[0] + ][age_band].values[0] if pd.isnull(mapped_value) and "Park home" in mapped_description: # We don't know enough in this case so we default to 0 @@ -563,7 +563,7 @@ def get_floor_u_value( insulation_lookup = s11[ s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type - ] + ] if insulation_lookup.empty: insulation_thickness = 0 else: