junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-12 13:29:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

amend etl code for new october data

Browse source
This commit is contained in:
Michael Duong 2025-11-01 15:34:45 +00:00
parent 0e0a473afe
commit 92fcbe8cdb
8 changed files with 295 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
etl/epc/DataProcessor.py
View file

@ -21,7 +21,7 @@ from etl.epc.settings import (
ENDING_SUFFIX_COMPONENT_COLS, ENDING_SUFFIX_COMPONENT_COLS,
POTENTIAL_COLUMNS, POTENTIAL_COLUMNS,
EFFICIENCY_FEATURES, EFFICIENCY_FEATURES,
DATA_ANOMALY_MATCHES DATA_ANOMALY_MATCHES,
) )
from recommendations.rdsap_tables import FLOOR_LEVEL_MAP from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
@ -249,7 +249,8 @@ class EPCDataProcessor:
# Map all anomaly values to None # Map all anomaly values to None
data_anomaly_map = dict( data_anomaly_map = dict(
zip( zip(
DATA_ANOMALY_MATCHES, [None] * len(DATA_ANOMALY_MATCHES), DATA_ANOMALY_MATCHES,
[None] * len(DATA_ANOMALY_MATCHES),
) )
) )
@ -749,6 +750,12 @@ class EPCDataProcessor:
self.data = self.data[~pd.isnull(self.data["HOTWATER_DESCRIPTION"])] self.data = self.data[~pd.isnull(self.data["HOTWATER_DESCRIPTION"])]
self.data = self.data[~pd.isnull(self.data["ROOF_DESCRIPTION"])] self.data = self.data[~pd.isnull(self.data["ROOF_DESCRIPTION"])]
# Remove any walls described as Basement walls since these are non-standard
# TODO: CHECK IF WE SHOULD MAP THESE U VALUES INSTEAD
index_to_remove = self.data["WALLS_DESCRIPTION"] == "Basement wall"
print(f"Removing {index_to_remove.sum()} records with basement walls")
self.data = self.data[~index_to_remove]
# Because park homes are surveyed unusually (for example, we don't have u-values to # Because park homes are surveyed unusually (for example, we don't have u-values to
# look up for their different components, they need to be collected in survey and aren't reflected in # look up for their different components, they need to be collected in survey and aren't reflected in
# EPCs) we'll ignore them from the model # EPCs) we'll ignore them from the model

10
etl/epc/Pipeline.py
View file

@ -112,6 +112,16 @@ clean_lookup["mainheatcont-description"] = new_mainheatcont_mapping.to_dict(
orient="records" orient="records"
) )
# TEMP FIX - GRANITE OR WHINSTONE BOOLEAN ISSUE
new_walls_description_mapping = pd.DataFrame(clean_lookup["walls-description"])
new_walls_description_mapping.loc[
new_walls_description_mapping["original_description"].str.contains("Granite"),
"is_granite_or_whinstone",
] = True
clean_lookup["walls-description"] = new_walls_description_mapping.to_dict(
orient="records"
)
class EPCPipeline: class EPCPipeline:
""" """

2
etl/epc/property_change_app.py
View file

@ -12,7 +12,7 @@ def main():
""" """
directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()] directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()]
# directories = directories[0:3] # directories = directories[235:275]
epc_pipeline = EPCPipeline( epc_pipeline = EPCPipeline(
directories=directories, directories=directories,

59
etl/epc_clean/epc_attributes/FloorAttributes.py
View file

@ -1,17 +1,26 @@
import re import re
from typing import Dict, Union from typing import Dict, Union
from BaseUtility import Definitions from BaseUtility import Definitions
from etl.epc_clean.epc_attributes.attribute_utils import extract_thermal_transmittance, extract_component_types from etl.epc_clean.epc_attributes.attribute_utils import (
extract_thermal_transmittance,
extract_component_types,
)
class FloorAttributes(Definitions): class FloorAttributes(Definitions):
DWELLING_BELOW = ["another dwelling below", "other premises below"] DWELLING_BELOW = ["another dwelling below", "other premises below"]
FLOOR_TYPES = ["assumed", "to unheated space", "to external air", "suspended", "solid"] FLOOR_TYPES = [
"assumed",
"to unheated space",
"to external air",
"suspended",
"solid",
]
# For the short term, while we are still exploring the data, we maintain a list of error cases which # For the short term, while we are still exploring the data, we maintain a list of error cases which
# we want to ignore and consider as no data. # we want to ignore and consider as no data.
OBSERVED_ERRORS = ["Conservatory", "insulated"] OBSERVED_ERRORS = ["Conservatory", "insulated", "Basement"]
WELSH_TEXT = { WELSH_TEXT = {
"(anheddiad arall islaw)": "(another dwelling below)", "(anheddiad arall islaw)": "(another dwelling below)",
@ -35,32 +44,40 @@ class FloorAttributes(Definitions):
"i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
"i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation", "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation",
"igçör awyr y tu allan, wedigçöi inswleiddio (rhagdybiaeth)": "to external air, insulated (assumed)", "igçör awyr y tu allan, wedigçöi inswleiddio (rhagdybiaeth)": "to external air, insulated (assumed)",
"crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)" "crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)",
} }
def __init__(self, description: str): def __init__(self, description: str):
self.description: str = description.lower() self.description: str = description.lower()
self.nodata = (not description) or (description in self.DATA_ANOMALY_MATCHES) or ( self.nodata = (
description in self.OBSERVED_ERRORS) or (self.description == "sap05:floor") (not description)
or (description in self.DATA_ANOMALY_MATCHES)
or (description in self.OBSERVED_ERRORS)
or (self.description == "sap05:floor")
)
# Try and perform a translation, incase it's in welsh # Try and perform a translation, incase it's in welsh
self.translate_welsh_text() self.translate_welsh_text()
if not self.nodata and not any( if not self.nodata and not any(
rt in self.description for rt in rt in self.description
self.FLOOR_TYPES + self.DWELLING_BELOW + ["average thermal transmittance"] for rt in self.FLOOR_TYPES
+ self.DWELLING_BELOW
+ ["average thermal transmittance"]
): ):
raise ValueError('Invalid description') raise ValueError("Invalid description")
def translate_welsh_text(self): def translate_welsh_text(self):
uvalue_match = re.search( uvalue_match = re.search(
r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k', self.description r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k",
self.description,
) )
uvalue_match2 = re.search( uvalue_match2 = re.search(
r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k",
self.description,
) )
# Step 2: Generalized translation with placeholder # Step 2: Generalized translation with placeholder
@ -69,7 +86,7 @@ class FloorAttributes(Definitions):
uvalue = uvalue_match.group(1) uvalue = uvalue_match.group(1)
else: else:
uvalue = uvalue_match2.group(1) uvalue = uvalue_match2.group(1)
self.description = f'average thermal transmittance {uvalue} w/m-¦k' self.description = f"average thermal transmittance {uvalue} w/m-¦k"
else: else:
translation = self.WELSH_TEXT.get(self.description) translation = self.WELSH_TEXT.get(self.description)
@ -89,11 +106,15 @@ class FloorAttributes(Definitions):
result, description = extract_thermal_transmittance(result, description) result, description = extract_thermal_transmittance(result, description)
# floor type # floor type
result, description = extract_component_types(result, description, list_of_components=self.FLOOR_TYPES) result, description = extract_component_types(
result, description, list_of_components=self.FLOOR_TYPES
)
# check if there is another dwelling below # check if there is another dwelling below
result['another_property_below'] = "(another dwelling below)" in description or "(other premises below)" in \ result["another_property_below"] = (
description "(another dwelling below)" in description
or "(other premises below)" in description
)
thickness_map = { thickness_map = {
"external insulation": "average", "external insulation": "average",
@ -102,17 +123,17 @@ class FloorAttributes(Definitions):
"partial insulation": "below average", "partial insulation": "below average",
"no insulation": "none", "no insulation": "none",
"additional insulation": "above average", "additional insulation": "above average",
"insulated": "average" "insulated": "average",
} }
for key, value in thickness_map.items(): for key, value in thickness_map.items():
if key in description: if key in description:
result['insulation_thickness'] = value result["insulation_thickness"] = value
break break
else: else:
result['insulation_thickness'] = None result["insulation_thickness"] = None
if result["another_property_below"]: if result["another_property_below"]:
result["thermal_transmittance"] = 0 result["thermal_transmittance"] = 0
result["thermal_transmittance_unit"] = 'w/m-¦k' result["thermal_transmittance_unit"] = "w/m-¦k"
return result return result

89
etl/epc_clean/epc_attributes/RoofAttributes.py
View file

@ -1,12 +1,28 @@
import re import re
from typing import Dict, Union from typing import Dict, Union
from BaseUtility import Definitions from BaseUtility import Definitions
from etl.epc_clean.epc_attributes.attribute_utils import extract_component_types, extract_thermal_transmittance from etl.epc_clean.epc_attributes.attribute_utils import (
extract_component_types,
extract_thermal_transmittance,
)
class RoofAttributes(Definitions): class RoofAttributes(Definitions):
ROOF_TYPES = ['pitched', 'roof room', 'loft', 'flat', 'thatched', 'at rafters', 'assumed'] ROOF_TYPES = [
DWELLING_ABOVE = ["another dwelling above", "other premises above", "other dwelling above"] "pitched",
"roof room",
"loft",
"flat",
"thatched",
"at rafters",
"assumed",
]
DWELLING_ABOVE = [
"another dwelling above",
"other premises above",
"other dwelling above",
"(same dwelling above)",
]
WELSH_TEXT = { WELSH_TEXT = {
"ar oleddf, dim inswleiddio": "pitched, no insulation", "ar oleddf, dim inswleiddio": "pitched, no insulation",
@ -18,10 +34,10 @@ class RoofAttributes(Definitions):
"ar oleddf, wedi?i inswleiddio": "pitched, insulated", "ar oleddf, wedi?i inswleiddio": "pitched, insulated",
"ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)",
"ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation",
"ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": "pitched, insulated at rafters",
"ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "ar oleddf, wedi?i inswleiddio wrth y trawstiau": "pitched, insulated at rafters",
"ar oleddf, wedi?i inswleiddio wrth y trawstia": 'pitched, insulated at rafters', "ar oleddf, wedi?i inswleiddio wrth y trawstia": "pitched, insulated at rafters",
"ar oleddf, wedigçöi inswleiddio wrth y trawstia": 'pitched, insulated at rafters', "ar oleddf, wedigçöi inswleiddio wrth y trawstia": "pitched, insulated at rafters",
"yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)",
"yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation",
"yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)",
@ -43,9 +59,18 @@ class RoofAttributes(Definitions):
} }
DEFAULT_KEYS = [ DEFAULT_KEYS = [
'thermal_transmittance', 'thermal_transmittance_unit', 'is_pitched', 'is_roof_room', "thermal_transmittance",
'is_loft', 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed', 'has_dwelling_above', "thermal_transmittance_unit",
'is_valid', 'insulation_thickness' "is_pitched",
"is_roof_room",
"is_loft",
"is_flat",
"is_thatched",
"is_at_rafters",
"is_assumed",
"has_dwelling_above",
"is_valid",
"insulation_thickness",
] ]
def __init__(self, description: str): def __init__(self, description: str):
@ -54,14 +79,21 @@ class RoofAttributes(Definitions):
""" """
self.description: str = description.lower().strip() self.description: str = description.lower().strip()
self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or self.description == "sap05:roof" self.nodata = (
not description
or description in self.DATA_ANOMALY_MATCHES
or self.description == "sap05:roof"
)
self.welsh_translation_search() self.welsh_translation_search()
if not self.nodata and not any( if not self.nodata and not any(
rt in self.description for rt in self.ROOF_TYPES + self.DWELLING_ABOVE + ["average thermal transmittance"] rt in self.description
for rt in self.ROOF_TYPES
+ self.DWELLING_ABOVE
+ ["average thermal transmittance"]
): ):
raise ValueError('Invalid description') raise ValueError("Invalid description")
def welsh_translation_search(self): def welsh_translation_search(self):
""" """
@ -76,7 +108,7 @@ class RoofAttributes(Definitions):
r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof",
r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof",
r"ar oleddf, (\d+mm) o inswleiddio yn y llofft", r"ar oleddf, (\d+mm) o inswleiddio yn y llofft",
r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft" r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft",
] ]
li_thickness_match = None li_thickness_match = None
for regex in loft_insulation_regexes: for regex in loft_insulation_regexes:
@ -84,9 +116,14 @@ class RoofAttributes(Definitions):
if li_thickness_match: if li_thickness_match:
break break
uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description) uvalue_search = re.search(
r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k",
self.description,
)
uvalue_search2 = re.search( uvalue_search2 = re.search(
r'trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k', self.description, re.IGNORECASE r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m.+k",
self.description,
re.IGNORECASE,
) )
# Step 2: Generalized translation with placeholder # Step 2: Generalized translation with placeholder
@ -121,9 +158,13 @@ class RoofAttributes(Definitions):
result, description = extract_thermal_transmittance(result, description) result, description = extract_thermal_transmittance(result, description)
# roof type # roof type
result, description = extract_component_types(result, description, list_of_components=self.ROOF_TYPES) result, description = extract_component_types(
result, description, list_of_components=self.ROOF_TYPES
)
result["has_dwelling_above"] = any([x in description for x in self.DWELLING_ABOVE]) result["has_dwelling_above"] = any(
[x in description for x in self.DWELLING_ABOVE]
)
for dwelling_above in self.DWELLING_ABOVE: for dwelling_above in self.DWELLING_ABOVE:
description = description.replace(dwelling_above, "") description = description.replace(dwelling_above, "")
@ -136,7 +177,7 @@ class RoofAttributes(Definitions):
# Search for a regular expression that matches 150 insulation # Search for a regular expression that matches 150 insulation
match = re.search(r"(\d+\+?)\s*insulation", description) match = re.search(r"(\d+\+?)\s*insulation", description)
if match: if match:
result['insulation_thickness'] = match.group(1) result["insulation_thickness"] = match.group(1)
# insulation thickness # insulation thickness
thickness_map = { thickness_map = {
@ -149,21 +190,21 @@ class RoofAttributes(Definitions):
} }
for key, value in thickness_map.items(): for key, value in thickness_map.items():
if key in description: if key in description:
result['insulation_thickness'] = value result["insulation_thickness"] = value
# Remove the match from the description # Remove the match from the description
# description = description.replace(key, "") # description = description.replace(key, "")
break break
# Extract insulation thickness in mm, if present # Extract insulation thickness in mm, if present
match = re.search(r'(\d+\+?)\s*mm', description) match = re.search(r"(\d+\+?)\s*mm", description)
if match: if match:
result['insulation_thickness'] = match.group(1) result["insulation_thickness"] = match.group(1)
if "insulation_thickness" not in result: if "insulation_thickness" not in result:
result['insulation_thickness'] = None result["insulation_thickness"] = None
if result["has_dwelling_above"]: if result["has_dwelling_above"]:
result["thermal_transmittance"] = 0 result["thermal_transmittance"] = 0
result["thermal_transmittance_unit"] = 'w/m-¦k' result["thermal_transmittance_unit"] = "w/m-¦k"
return result return result

146
etl/epc_clean/epc_attributes/WallAttributes.py
View file

@ -3,76 +3,78 @@ from typing import Dict, Union
from BaseUtility import Definitions from BaseUtility import Definitions
from etl.epc_clean.epc_attributes.attribute_utils import ( from etl.epc_clean.epc_attributes.attribute_utils import (
extract_component_types, extract_component_types,
extract_thermal_transmittance extract_thermal_transmittance,
) )
class WallAttributes(Definitions): class WallAttributes(Definitions):
WALL_TYPES = ['cavity wall', 'filled cavity', 'solid brick', 'system built', 'timber frame', 'granite or whinstone', WALL_TYPES = [
'as built', 'cob', 'assumed', 'sandstone or limestone', "park home"] "cavity wall",
"filled cavity",
"solid brick",
"system built",
"timber frame",
"granite or whinstone",
"as built",
"cob",
"assumed",
"sandstone or limestone",
"park home",
]
WELSH_TEXT = { WELSH_TEXT = {
"Briciau solet, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Briciau solet, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Solid brick, as built, no insulation (assumed)",
"Solid brick, as built, no insulation (assumed)", "Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "Cavity wall, as built, partial insulation (assumed)",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)': "Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol": "Cavity wall, as built, partial insulation",
'Cavity wall, as built, partial insulation (assumed)', "Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Cavity wall, as built, no insulation (assumed)",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, inswleiddio rhannol': "Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Cavity wall, as built, no insulation",
'Cavity wall, as built, partial insulation', "Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Sandstone or limestone, as built, no insulation (assumed)",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': "Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Sandstone or limestone, as built, no insulation",
'Cavity wall, as built, no insulation (assumed)', "Waliau ceudod, ceudod wediGÇÖi lenwi": "Cavity wall, filled cavity",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, dim inswleiddio': "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Cavity wall, as built, insulated (assumed)",
'Cavity wall, as built, no insulation', "Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Cavity wall, as built, insulated",
'Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': "Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Granite or whinstone, as built, no insulation (assumed)",
'Sandstone or limestone, as built, no insulation (assumed)', "Waliau ceudod,": "Cavity wall, as built, no insulation",
'Tywodfaen, fel yGÇÖu hadeiladwyd, dim inswleiddio': "Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "Timber frame, as built, insulated (assumed)",
'Sandstone or limestone, as built, no insulation', "Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "Timber frame, as built, insulated",
'Waliau ceudod, ceudod wediGÇÖi lenwi': 'Cavity wall, filled cavity', "Gwenithfaen neu risgraig, gydag inswleiddio allanol": "Granite or whinstone, with external insulation",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)': "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "System built, as built, no insulation (assumed)",
'Cavity wall, as built, insulated (assumed)', "Tywodfaen, gydag inswleiddio mewnol": "Sandstone or limestone, with internal insulation",
'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio': "Waliau ceudod, ynysydd allanol a llenwi ceudod": "Cavity wall, filled cavity and external insulation",
'Cavity wall, as built, insulated', "Gwenithfaen neu risgraig, gydag inswleiddio mewnol": "Granite or whinstone, with internal insulation",
'Gwenithfaen neu risgraig, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': "Ffr+óm bren, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "Timber frame, as built, partial insulation (assumed)",
'Granite or whinstone, as built, no insulation (assumed)', "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)": "System built, as built, insulated (assumed)",
'Waliau ceudod,': 'Cavity wall, as built, no insulation', "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio": "System built, as built, insulated",
'Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)': "WediGÇÖu hadeiladu yn +¦l system, gydag inswleiddio allanol": "System built, with external insulation",
'Timber frame, as built, insulated (assumed)', "Briciau solet, gydag inswleiddio mewnol": "Solid brick, with internal insulation",
'Ffr+óm bren, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio': "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)": "System built, as built, partial insulation (assumed)",
'Timber frame, as built, insulated', "WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol": "System built, as built, partial insulation",
'Gwenithfaen neu risgraig, gydag inswleiddio allanol': 'Granite or whinstone, with external insulation', "Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)": "Timber frame, as built, no insulation (assumed)",
'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)': "Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio": "Timber frame, as built, no insulation",
'System built, as built, no insulation (assumed)', "Tywodfaen, gydag inswleiddio allanol": "Sandstone or limestone, with external insulation",
'Tywodfaen, gydag inswleiddio mewnol': 'Sandstone or limestone, with internal insulation', "Waliau ceudod, gydag inswleiddio allanol": "Cavity wall, with external insulation",
'Waliau ceudod, ynysydd allanol a llenwi ceudod': 'Cavity wall, filled cavity and external insulation', "Briciau solet, gydag inswleiddio allanol": "Solid brick, with external insulation",
'Gwenithfaen neu risgraig, gydag inswleiddio mewnol': 'Granite or whinstone, with internal insulation',
'Ffr+óm bren, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)':
'Timber frame, as built, partial insulation (assumed)',
'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)':
'System built, as built, insulated (assumed)',
'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio':
'System built, as built, insulated',
'WediGÇÖu hadeiladu yn +¦l system, gydag inswleiddio allanol': 'System built, with external insulation',
'Briciau solet, gydag inswleiddio mewnol': 'Solid brick, with internal insulation',
'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol (rhagdybiaeth)':
'System built, as built, partial insulation (assumed)',
'WediGÇÖu hadeiladu yn +¦l system, fel yGÇÖu hadeiladwyd, inswleiddio rhannol':
'System built, as built, partial insulation',
'Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio (rhagdybiaeth)':
'Timber frame, as built, no insulation (assumed)',
'Ffr+óm bren, fel yGÇÖu hadeiladwyd, dim inswleiddio':
'Timber frame, as built, no insulation',
'Tywodfaen, gydag inswleiddio allanol': 'Sandstone or limestone, with external insulation',
'Waliau ceudod, gydag inswleiddio allanol': 'Cavity wall, with external insulation',
'Briciau solet, gydag inswleiddio allanol': 'Solid brick, with external insulation',
# Add in some corrections: # Add in some corrections:
'Co with external insulation': 'Cob, with external insulation', "Co with external insulation": "Cob, with external insulation",
'Cowith external insulation': 'Cob, with external insulation', "Cowith external insulation": "Cob, with external insulation",
} }
DEFAULT_KEYS = [ DEFAULT_KEYS = [
'thermal_transmittance', 'thermal_transmittance_unit', 'is_cavity_wall', 'is_filled_cavity', "thermal_transmittance",
'is_solid_brick', 'is_system_built', 'is_timber_frame', 'is_granite_or_whinstone', "thermal_transmittance_unit",
'is_as_built', 'is_cob', 'is_assumed', 'is_sandstone_or_limestone', "is_cavity_wall",
'insulation_thickness', 'external_insulation', 'internal_insulation' "is_filled_cavity",
"is_solid_brick",
"is_system_built",
"is_timber_frame",
"is_granite_or_whinstone",
"is_as_built",
"is_cob",
"is_assumed",
"is_sandstone_or_limestone",
"insulation_thickness",
"external_insulation",
"internal_insulation",
] ]
CORRECTIONS = { CORRECTIONS = {
@ -98,7 +100,9 @@ class WallAttributes(Definitions):
:return: :return:
""" """
uvalue_search = re.search(r"Trawsyriannedd thermol cyfartalog (\d+\.?\d*)", self.description) uvalue_search = re.search(
r"Trawsyriannedd thermol cyfartalog (\d+\.?\d*)", self.description
)
if uvalue_search: if uvalue_search:
uvalue = uvalue_search.group(1) uvalue = uvalue_search.group(1)
@ -123,7 +127,9 @@ class WallAttributes(Definitions):
result, description = extract_thermal_transmittance(result, description) result, description = extract_thermal_transmittance(result, description)
# wall type # wall type
result, description = extract_component_types(result, description, list_of_components=self.WALL_TYPES) result, description = extract_component_types(
result, description, list_of_components=self.WALL_TYPES
)
# Handle some edge cases # Handle some edge cases
if "sandstone" in description and not result["is_sandstone_or_limestone"]: if "sandstone" in description and not result["is_sandstone_or_limestone"]:
@ -137,18 +143,18 @@ class WallAttributes(Definitions):
"partial insulation": "below average", "partial insulation": "below average",
"no insulation": "none", "no insulation": "none",
"additional insulation": "above average", "additional insulation": "above average",
"insulated": "average" "insulated": "average",
} }
for key, value in thickness_map.items(): for key, value in thickness_map.items():
if key in description: if key in description:
result['insulation_thickness'] = value result["insulation_thickness"] = value
break break
else: else:
result['insulation_thickness'] = None result["insulation_thickness"] = None
# insulation type # insulation type
result['external_insulation'] = 'external insulation' in description result["external_insulation"] = "external insulation" in description
result['internal_insulation'] = 'internal insulation' in description result["internal_insulation"] = "internal insulation" in description
if result["is_filled_cavity"]: if result["is_filled_cavity"]:
# If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation # If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation
@ -159,7 +165,11 @@ class WallAttributes(Definitions):
else: else:
result["insulation_thickness"] = "average" result["insulation_thickness"] = "average"
if result["is_cavity_wall"] & result["is_as_built"] & (result["insulation_thickness"] == "average"): if (
result["is_cavity_wall"]
& result["is_as_built"]
& (result["insulation_thickness"] == "average")
):
result["is_filled_cavity"] = True result["is_filled_cavity"] = True
return result return result

102
recommendations/rdsap_tables.py
View file

@ -98,6 +98,13 @@ age_band_data = [
"Northern_Ireland": "2014-2022", "Northern_Ireland": "2014-2022",
"Park_home_UK": None, "Park_home_UK": None,
}, },
{
"age_band": "M",
"England_Wales": "2022 onwards",
"Scotland": "2024 onwards",
"Northern_Ireland": "2023 onwards",
"Park_home_UK": None,
},
] ]
england_wales_age_band_lookup = { england_wales_age_band_lookup = {
@ -123,6 +130,7 @@ default_wall_thickness = [
"J": 450, "J": 450,
"K": 450, "K": 450,
"L": 450, "L": 450,
"M": 450,
}, },
{ {
"type": "solid brick", "type": "solid brick",
@ -138,6 +146,7 @@ default_wall_thickness = [
"J": 300, "J": 300,
"K": 300, "K": 300,
"L": 300, "L": 300,
"M": 300,
}, },
{ {
"type": "cavity", "type": "cavity",
@ -153,6 +162,7 @@ default_wall_thickness = [
"J": 300, "J": 300,
"K": 300, "K": 300,
"L": 300, "L": 300,
"M": 300,
}, },
{ {
"type": "timber frame", "type": "timber frame",
@ -168,6 +178,7 @@ default_wall_thickness = [
"J": 300, "J": 300,
"K": 300, "K": 300,
"L": 300, "L": 300,
"M": 300,
}, },
{ {
"type": "cob", "type": "cob",
@ -183,6 +194,7 @@ default_wall_thickness = [
"J": 590, "J": 590,
"K": 590, "K": 590,
"L": 590, "L": 590,
"M": 590,
}, },
{ {
"type": "system build", "type": "system build",
@ -198,6 +210,7 @@ default_wall_thickness = [
"J": 300, "J": 300,
"K": 300, "K": 300,
"L": 300, "L": 300,
"M": 300,
}, },
{ {
"type": "park home", "type": "park home",
@ -213,6 +226,7 @@ default_wall_thickness = [
"J": 100, "J": 100,
"K": 100, "K": 100,
"L": 100, "L": 100,
"M": 100,
}, },
] ]
@ -253,8 +267,36 @@ wall_types = [
] ]
u_values = [ u_values = [
["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], [
["a", "a", "a", "a", "1.7b", "1.0", "0.6", "0.60", "0.45", "0.35", "0.30", "0.28"], "a",
"a",
"a",
"a",
"1.7b",
"1.0",
"0.6",
"0.60",
"0.45",
"0.35",
"0.30",
"0.28",
"0.26",
],
[
"a",
"a",
"a",
"a",
"1.7b",
"1.0",
"0.6",
"0.60",
"0.45",
"0.35",
"0.30",
"0.28",
"0.26",
],
[ [
"1.7", "1.7",
"1.7", "1.7",
@ -268,6 +310,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.55", "0.55",
@ -282,6 +325,7 @@ u_values = [
"0.25", "0.25",
"0.21", "0.21",
"0.21", "0.21",
"0.20",
], ],
[ [
"0.32", "0.32",
@ -296,6 +340,7 @@ u_values = [
"0.19", "0.19",
"0.17", "0.17",
"0.16", "0.16",
"0.15",
], ],
[ [
"0.23", "0.23",
@ -310,6 +355,7 @@ u_values = [
"0.15", "0.15",
"0.14", "0.14",
"0.14", "0.14",
"0.13",
], ],
[ [
"0.18", "0.18",
@ -324,6 +370,7 @@ u_values = [
"0.13", "0.13",
"0.12", "0.12",
"0.12", "0.12",
"0.11",
], ],
[ [
"0.80", "0.80",
@ -338,6 +385,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.40", "0.40",
@ -352,6 +400,7 @@ u_values = [
"0.25", "0.25",
"0.21", "0.21",
"0.21", "0.21",
"0.20",
], ],
[ [
"0.26", "0.26",
@ -366,6 +415,7 @@ u_values = [
"0.19", "0.19",
"0.17", "0.17",
"0.16", "0.16",
"0.15",
], ],
[ [
"0.20", "0.20",
@ -380,6 +430,7 @@ u_values = [
"0.15", "0.15",
"0.14", "0.14",
"0.14", "0.14",
"0.13",
], ],
[ [
"0.16", "0.16",
@ -394,6 +445,7 @@ u_values = [
"0.13", "0.13",
"0.12", "0.12",
"0.12", "0.12",
"0.11",
], ],
[ [
"1.5", "1.5",
@ -408,6 +460,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.53", "0.53",
@ -422,6 +475,7 @@ u_values = [
"0.25", "0.25",
"0.21", "0.21",
"0.21", "0.21",
"0.20",
], ],
[ [
"0.32", "0.32",
@ -436,6 +490,7 @@ u_values = [
"0.19", "0.19",
"0.17", "0.17",
"0.16", "0.16",
"0.15",
], ],
[ [
"0.23", "0.23",
@ -450,6 +505,7 @@ u_values = [
"0.15", "0.15",
"0.14", "0.14",
"0.14", "0.14",
"0.13",
], ],
[ [
"0.18", "0.18",
@ -464,6 +520,7 @@ u_values = [
"0.13", "0.13",
"0.12", "0.12",
"0.12", "0.12",
"0.11",
], ],
[ [
"0.7", "0.7",
@ -478,6 +535,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.37", "0.37",
@ -492,6 +550,7 @@ u_values = [
"0.25", "0.25",
"0.21", "0.21",
"0.21", "0.21",
"0.20",
], ],
[ [
"0.25", "0.25",
@ -506,6 +565,7 @@ u_values = [
"0.19", "0.19",
"0.17", "0.17",
"0.16", "0.16",
"0.15",
], ],
[ [
"0.19", "0.19",
@ -520,6 +580,7 @@ u_values = [
"0.15", "0.15",
"0.14", "0.14",
"0.14", "0.14",
"0.13",
], ],
[ [
"0.16", "0.16",
@ -534,6 +595,7 @@ u_values = [
"0.13", "0.13",
"0.12", "0.12",
"0.12", "0.12",
"0.11",
], ],
[ [
"2.5", "2.5",
@ -548,6 +610,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.60", "0.60",
@ -562,6 +625,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"2.0", "2.0",
@ -576,6 +640,7 @@ u_values = [
"0.35", "0.35",
"0.30", "0.30",
"0.28", "0.28",
"0.26",
], ],
[ [
"0.60", "0.60",
@ -590,6 +655,7 @@ u_values = [
"0.25", "0.25",
"0.21", "0.21",
"0.21", "0.21",
"0.20",
], ],
[ [
"0.35", "0.35",
@ -604,6 +670,7 @@ u_values = [
"0.19", "0.19",
"0.17", "0.17",
"0.16", "0.16",
"0.15",
], ],
[ [
"0.25", "0.25",
@ -618,6 +685,7 @@ u_values = [
"0.15", "0.15",
"0.14", "0.14",
"0.14", "0.14",
"0.13",
], ],
[ [
"0.18", "0.18",
@ -632,10 +700,11 @@ u_values = [
"0.13", "0.13",
"0.12", "0.12",
"0.12", "0.12",
"0.11",
], ],
] ]
age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"] age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"]
wall_uvalues = [] wall_uvalues = []
for i, wall_type in enumerate(wall_types): for i, wall_type in enumerate(wall_types):
@ -645,13 +714,23 @@ for i, wall_type in enumerate(wall_types):
wall_uvalues.append(row) wall_uvalues.append(row)
parkhome_wall_uvalues = [ parkhome_wall_uvalues = [
{"Wall_type": "Park home as built", "F": "1.7", "G": "1.2", "I": "0.7", "K": "0.6"}, {
"Wall_type": "Park home as built",
"F": "1.7",
"G": "1.2",
"I": "0.7",
"K": "0.6",
"L": "0.6",
"M": "0.6",
},
{ {
"Wall_type": "Park home with additional insulation", "Wall_type": "Park home with additional insulation",
"F": "s1.1.2", "F": "s1.1.2",
"G": "s1.1.2", "G": "s1.1.2",
"I": "s1.1.2", "I": "s1.1.2",
"K": "s1.1.2", "K": "s1.1.2",
"L": "s1.1.2",
"M": "s1.1.2",
}, },
] ]
@ -713,14 +792,13 @@ epc_wall_description_map = {
# Granite/whinstone wall mappings # Granite/whinstone wall mappings
############################ ############################
"Granite or whinstone, as built, no insulation": "Stone: granite or whinstone as built", "Granite or whinstone, as built, no insulation": "Stone: granite or whinstone as built",
"Granite or whinstone, with internal insulation": "Stone/solid brick with 100 mm external or internal " "Granite or whinstone, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation",
"insulation", "Granite or whin, with internal insulation": "Stone/solid brick with 100 mm external or internal insulation",
"Granite or whinstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal " "Granite or whinstone, as built, partial insulation": "Stone/solid brick with 50 mm external or internal insulation",
"insulation", "Granite or whinstone, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation",
"Granite or whinstone, as built, insulated": "Stone/solid brick with 100 mm external or internal " "Granite or whin, as built, insulated": "Stone/solid brick with 100 mm external or internal insulation",
"insulation", "Granite or whinstone, with external insulation": "Stone/solid brick with 100 mm external or internal insulation",
"Granite or whinstone, with external insulation": "Stone/solid brick with 100 mm external or internal " "Granite or whin, with external insulation": "Stone/solid brick with 100 mm external or internal insulation",
"insulation",
############################ ############################
# System built wall mappings # System built wall mappings
############################ ############################

4
recommendations/recommendation_utils.py
View file

@ -205,7 +205,7 @@ def get_wall_u_value(
mapped_value = wall_uvalues_df[ mapped_value = wall_uvalues_df[
wall_uvalues_df["Wall_type"] == mapped_description wall_uvalues_df["Wall_type"] == mapped_description
][age_band].values[0] ][age_band].values[0]
if pd.isnull(mapped_value) and "Park home" in mapped_description: if pd.isnull(mapped_value) and "Park home" in mapped_description:
# We don't know enough in this case so we default to 0 # We don't know enough in this case so we default to 0
@ -563,7 +563,7 @@ def get_floor_u_value(
insulation_lookup = s11[ insulation_lookup = s11[
s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] s11["Age_band"].str.contains(age_band) & s11["Floor_construction"]
== floor_type == floor_type
] ]
if insulation_lookup.empty: if insulation_lookup.empty:
insulation_thickness = 0 insulation_thickness = 0
else: else: