diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 3b05c6ac..ca0e1cd9 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 0565ab94..85a02b61 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -86,7 +86,7 @@ def create_recommendation_scoring_data( } # Set staring u-values if we don't have them - if not scoring_dict["walls_thermal_transmittance"]: + if scoring_dict["walls_thermal_transmittance"] is None: scoring_dict["walls_thermal_transmittance"] = get_wall_u_value( clean_description=property.walls["clean_description"], age_band=property.age_band, @@ -94,7 +94,7 @@ def create_recommendation_scoring_data( is_sandstone_or_limestone=property.walls["is_sandstone_or_limestone"] ) - if not scoring_dict["floor_thermal_transmittance"]: + if scoring_dict["floor_thermal_transmittance"] is None: scoring_dict["floor_thermal_transmittance"] = get_floor_u_value( floor_type=property.floor_type, area=property.floor_area, @@ -104,7 +104,7 @@ def create_recommendation_scoring_data( age_band=property.age_band, ) - if not scoring_dict["roof_thermal_transmittance"]: + if scoring_dict["roof_thermal_transmittance"] is None: scoring_dict["roof_thermal_transmittance"] = get_roof_u_value( insulation_thickness=property.roof["insulation_thickness"], has_dwelling_above=property.roof["has_dwelling_above"], @@ -130,7 +130,7 @@ def create_recommendation_scoring_data( scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"] scoring_dict["walls_insulation_thickness_ENDING"] = "above average" else: - if not scoring_dict["walls_thermal_transmittance_ENDING"]: + if scoring_dict["walls_thermal_transmittance_ENDING"] is None: scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value( clean_description=property.walls["clean_description"], age_band=property.age_band, @@ -151,7 +151,7 @@ def create_recommendation_scoring_data( # We don't really see above average for this in the training data scoring_dict["floor_insulation_thickness_ENDING"] = "average" else: - if not scoring_dict["floor_thermal_transmittance_ENDING"]: + if scoring_dict["floor_thermal_transmittance_ENDING"] is None: scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value( floor_type=property.floor_type, area=property.floor_area, @@ -167,7 +167,7 @@ def create_recommendation_scoring_data( if recommendation["type"] not in ["wall_insulation", "floor_insulation"]: raise NotImplementedError("Implement me") - if not scoring_dict["roof_thermal_transmittance_ENDING"]: + if scoring_dict["roof_thermal_transmittance_ENDING"] is None: scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value( insulation_thickness=property.roof["insulation_thickness"], has_dwelling_above=property.roof["has_dwelling_above"], @@ -180,7 +180,7 @@ def create_recommendation_scoring_data( is_at_rafters=property.roof["is_at_rafters"], ) - if scoring_dict["roof_insulation_thickness_ENDING"] is None: - scoring_dict["roof_insulation_thickness_ENDING"] = "none" + if scoring_dict["roof_insulation_thickness_ENDING"] is None: + scoring_dict["roof_insulation_thickness_ENDING"] = "none" return scoring_dict diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 4f36dbe8..c11a0fae 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -1,15 +1,16 @@ from backend.Property import Property from etl.epc.DataProcessor import DataProcessor -from backend.app.plan.utils import create_recommendation_scoring_data +from backend.app.plan.utils import create_recommendation_scoring_data, get_cleaned from etl.epc.settings import COLUMNS_TO_MERGE_ON from epc_api.client import EpcClient import pandas as pd -import os import pytest -import pickle + +from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm +# import pickle # with open("sap_change_dataset.pickle", "rb") as f: # sap_change_dataset = pickle.load(f) # @@ -109,18 +110,14 @@ from tqdm import tqdm class TestSapModelPrep: @pytest.fixture - def cleaned(self): - with open( - os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaned.pickle", "rb" - ) as f: - return pickle.load(f) + def cleaning_data(self): + return read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", + ) @pytest.fixture - def cleaning_data(self): - with open( - os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaning_data.pickle", "rb" - ) as f: - return pickle.load(f) + def cleaned(self): + return get_cleaned() def test_fill_cavity_wall(self, cleaned, cleaning_data): """ @@ -395,7 +392,7 @@ class TestSapModelPrep: 'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0, 'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307, - 'walls_thermal_transmittance': 0.45, 'is_cavity_wall': False, 'is_filled_cavity': False, + 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', @@ -482,7 +479,7 @@ class TestSapModelPrep: ending_lodgement_date2 = '2020-11-24' - starting_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) + ending_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) recommendation2 = { "recommendation_id": 0, diff --git a/recommendations/tests/test_data/wall_uvalue_test_cases.py b/recommendations/tests/test_data/wall_uvalue_test_cases.py index 1cc6823c..e0c6ebe3 100644 --- a/recommendations/tests/test_data/wall_uvalue_test_cases.py +++ b/recommendations/tests/test_data/wall_uvalue_test_cases.py @@ -4,21 +4,21 @@ wall_uvalue_test_cases = [ "age_band": "A", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.7 + "uvalue": 1.3 }, { "clean_description": "Cavity wall, as built, partial insulation", "age_band": "F", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.4 + "uvalue": 0.85 }, { "clean_description": "Cavity wall, as built, partial insulation", - "age_band": "F", + "age_band": "G", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.4 + "uvalue": 0.5375 }, { diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index f3df168f..f34bbe81 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -118,6 +118,7 @@ class TestFloorRecommendations: input_properties[2].age_band = "A" input_properties[2].perimeter = 20 input_properties[2].wall_type = "solid brick" + input_properties[2].floor_type = "suspended" recommender = FloorRecommendations( property_instance=input_properties[2], @@ -160,6 +161,7 @@ class TestFloorRecommendations: input_properties[4].age_band = "B" input_properties[4].perimeter = 50 input_properties[4].wall_type = "solid brick" + input_properties[4].floor_type = "solid" recommender = FloorRecommendations( property_instance=input_properties[4], diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index d4ce5884..3663364c 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -407,36 +407,8 @@ class TestWallRecommendationsBase: wall_recommendations_instance.property.data = {"property-type": "house"} assert wall_recommendations_instance.ewi_valid is True - def test_recommend_without_u_value(self, wall_recommendations_instance): - wall_recommendations_instance.property.walls = { - "thermal_transmittance": None, - "is_solid_brick": False, - "is_cavity_wall": False, - "insulation_thickness": "none", - "clean_description": "Solid brick, as built, no insulation", - "is_granite_or_whinstone": False, - "is_sandstone_or_limestone": False, - } - wall_recommendations_instance.property.age_band = "A" - with pytest.raises(NotImplementedError): - wall_recommendations_instance.recommend() - class TestCavityWallRecommensations: - data = { - 'low-energy-fixed-light-count': '', 'address': '123 Fake Street', - 'floor-height': '', 'construction-age-band': 'England and Wales: 1950-1966', - 'address3': '', 'property-type': 'House', 'local-authority-label': 'Melton', - 'county': 'Leicestershire', 'postcode': 'LE14 2QH', - 'solar-water-heating-flag': 'N', 'constituency': 'E14000909', - 'number-heated-rooms': '5', 'local-authority': 'E07000133', 'built-form': 'End-Terrace', - 'address1': '1, 23 fake', 'total-floor-area': '85.0', 'environment-impact-current': '49', - 'number-habitable-rooms': 3, 'address2': 'Fake', 'posttown': 'IDK', - 'walls-energy-eff': 'Poor', 'current-energy-rating': 'D', - 'transaction-type': 'ECO assessment', 'uprn': '999', 'current-energy-efficiency': '57', - 'lodgement-date': '2019-07-10', 'lmk-key': '999', 'tenure': 'rental (private)', 'floor-level': 'NODATA!', - 'walls-description': 'Cavity wall, as built, no insulation (assumed)', - } def test_fill_empty_cavity(self): input_property = Property(id=1, postcode="F4k3", address1="123 fake street", epc_client=Mock()) @@ -465,10 +437,10 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.5 - assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.25) + assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.37) assert np.isclose(recommender.recommendations[0]["cost"], 1000) - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.26) + assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.38) assert np.isclose(recommender.recommendations[1]["cost"], 1250) def test_fill_partial_filled_cavity(self): @@ -498,10 +470,10 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.3 - assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.56) + assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.43) assert np.isclose(recommender.recommendations[0]["cost"], 1000) - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.57) + assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.45) assert np.isclose(recommender.recommendations[1]["cost"], 1250) def test_system_built_wall(self):