diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index c500c095..cb3de9f4 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -572,6 +572,17 @@ class DataProcessor: @staticmethod def clean_efficiency_variables(df): + + """ + These is scope to clean this by the model per corresponding description. + E.g. for WALLS_ENG_EFF we could look at the mode efficiency rating by description and + fill in the missing values with this. + When looking at this initially, there are a large volume of records with missing energy efficiency + values and therefore a simpler approach was taken just to test including these variables + :param df: + :return: + """ + missings = pd.isnull(df).sum() missings = missings[missings >= 1] diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 12f6f0c0..300c0417 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -258,6 +258,13 @@ def make_uvalues(df): # Roof # ~~~~~~~~~~~~~~~~~~ + if x["has_dwelling_above"]: + if x["roof_thermal_transmittance"] != 0: + raise ValueError("Should have 0 u-value for roof") + + if x["roof_thermal_transmittance_ENDING"] != 0: + raise ValueError("Should have 0 u-value for roof") + starting_roof_uvalue = x["roof_thermal_transmittance"] if pd.isnull(starting_roof_uvalue): starting_roof_uvalue = get_roof_u_value( @@ -301,6 +308,11 @@ def make_uvalues(df): wall_type = get_wall_type(**x) if x["another_property_below"]: + if x["floor_thermal_transmittance"] != 0: + raise ValueError("Should have 0 u-value for floor") + + if x["floor_thermal_transmittance_ENDING"] != 0: + raise ValueError("Should have 0 u-value for floor") starting_floor_uvalue, ending_floor_uvalue = 0, 0 else: starting_floor_uvalue = x["floor_thermal_transmittance"] diff --git a/etl/epc_clean/epc_attributes/WallAttributes.py b/etl/epc_clean/epc_attributes/WallAttributes.py index 03fe6d67..40a5d5db 100644 --- a/etl/epc_clean/epc_attributes/WallAttributes.py +++ b/etl/epc_clean/epc_attributes/WallAttributes.py @@ -133,4 +133,13 @@ class WallAttributes(Definitions): result['external_insulation'] = 'external insulation' in description result['internal_insulation'] = 'internal insulation' in description + if result["is_filled_cavity"]: + # If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation + if result["external_insulation"]: + result["insulation_thickness"] = "above average" + elif result["internal_insulation"]: + result["insulation_thickness"] = "above average" + else: + result["insulation_thickness"] = "average" + return result diff --git a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py index 5738f77f..280e7459 100644 --- a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py @@ -1,14 +1,14 @@ clean_floor_cases = [ - {'original_description': '(another dwelling below)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False, + {'original_description': '(another dwelling below)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, - {'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False, + {'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, - {'original_description': '(other premises below)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(other premises below)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, @@ -342,8 +342,8 @@ clean_floor_cases = [ {'original_description': 'To unheated space, no insulation (assumed)', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False}, - {'original_description': '(eiddo arall islaw)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(eiddo arall islaw)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, diff --git a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py index ee7f865b..6b719afd 100644 --- a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py @@ -1,10 +1,11 @@ clean_roof_test_cases = [ - {'original_description': '(another dwelling above)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, + {'original_description': '(another dwelling above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, + 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, - {'original_description': '(other premises above)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(other premises above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, @@ -362,8 +363,9 @@ clean_roof_test_cases = [ 'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'}, - {'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, + {'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, + 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, {'original_description': 'Ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None, diff --git a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py index 40d6fb9c..300702a7 100644 --- a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py @@ -567,17 +567,17 @@ wall_cases = [ {'original_description': 'Cavity wall, filled cavity', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None, + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average", 'external_insulation': False, 'internal_insulation': False}, {'original_description': 'Cavity wall, filled cavity and external insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': True, 'internal_insulation': False}, {'original_description': 'Cavity wall, filled cavity and internal insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': False, 'internal_insulation': True}, {'original_description': 'Cavity wall, with external insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': False, 'is_solid_brick': False, @@ -723,7 +723,7 @@ wall_cases = [ {'original_description': 'Waliau ceudod, ceudod wediGÇÖi lenwi', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None, + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average", 'external_insulation': False, 'internal_insulation': False}, {'original_description': 'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)', 'thermal_transmittance': None, @@ -778,7 +778,7 @@ wall_cases = [ {'original_description': 'Waliau ceudod, ynysydd allanol a llenwi ceudod', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': True, 'internal_insulation': False}, {'original_description': 'Gwenithfaen neu risgraig, gydag inswleiddio mewnol', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, diff --git a/etl/epc_clean/tests/test_roof_attributes.py b/etl/epc_clean/tests/test_roof_attributes.py index 54b59f1a..b0663a3e 100644 --- a/etl/epc_clean/tests/test_roof_attributes.py +++ b/etl/epc_clean/tests/test_roof_attributes.py @@ -75,8 +75,8 @@ class TestRoofAttributes: "is_assumed": False, "is_flat": False, "is_thatched": False, - "thermal_transmittance": None, - "thermal_transmittance_unit": None, + "thermal_transmittance": 0, + "thermal_transmittance_unit": "w/m-¦k", } for k in expected_output: