Fixed broken tests

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/plan/utils.py

@@ -86,7 +86,7 @@ def create_recommendation_scoring_data(
     }
 
     # Set staring u-values if we don't have them
-    if not scoring_dict["walls_thermal_transmittance"]:
+    if scoring_dict["walls_thermal_transmittance"] is None:
         scoring_dict["walls_thermal_transmittance"] = get_wall_u_value(
             clean_description=property.walls["clean_description"],
             age_band=property.age_band,
@@ -94,7 +94,7 @@ def create_recommendation_scoring_data(
             is_sandstone_or_limestone=property.walls["is_sandstone_or_limestone"]
         )
 
-    if not scoring_dict["floor_thermal_transmittance"]:
+    if scoring_dict["floor_thermal_transmittance"] is None:
         scoring_dict["floor_thermal_transmittance"] = get_floor_u_value(
             floor_type=property.floor_type,
             area=property.floor_area,
@@ -104,7 +104,7 @@ def create_recommendation_scoring_data(
             age_band=property.age_band,
         )
 
-    if not scoring_dict["roof_thermal_transmittance"]:
+    if scoring_dict["roof_thermal_transmittance"] is None:
         scoring_dict["roof_thermal_transmittance"] = get_roof_u_value(
             insulation_thickness=property.roof["insulation_thickness"],
             has_dwelling_above=property.roof["has_dwelling_above"],
@@ -130,7 +130,7 @@ def create_recommendation_scoring_data(
         scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"]
         scoring_dict["walls_insulation_thickness_ENDING"] = "above average"
     else:
-        if not scoring_dict["walls_thermal_transmittance_ENDING"]:
+        if scoring_dict["walls_thermal_transmittance_ENDING"] is None:
             scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value(
                 clean_description=property.walls["clean_description"],
                 age_band=property.age_band,
@@ -151,7 +151,7 @@ def create_recommendation_scoring_data(
         # We don't really see above average for this in the training data
         scoring_dict["floor_insulation_thickness_ENDING"] = "average"
     else:
-        if not scoring_dict["floor_thermal_transmittance_ENDING"]:
+        if scoring_dict["floor_thermal_transmittance_ENDING"] is None:
             scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value(
                 floor_type=property.floor_type,
                 area=property.floor_area,
@@ -167,7 +167,7 @@ def create_recommendation_scoring_data(
     if recommendation["type"] not in ["wall_insulation", "floor_insulation"]:
         raise NotImplementedError("Implement me")
 
-    if not scoring_dict["roof_thermal_transmittance_ENDING"]:
+    if scoring_dict["roof_thermal_transmittance_ENDING"] is None:
         scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value(
             insulation_thickness=property.roof["insulation_thickness"],
             has_dwelling_above=property.roof["has_dwelling_above"],
@@ -180,7 +180,7 @@ def create_recommendation_scoring_data(
             is_at_rafters=property.roof["is_at_rafters"],
         )
 
-        if scoring_dict["roof_insulation_thickness_ENDING"] is None:
-            scoring_dict["roof_insulation_thickness_ENDING"] = "none"
+    if scoring_dict["roof_insulation_thickness_ENDING"] is None:
+        scoring_dict["roof_insulation_thickness_ENDING"] = "none"
 
     return scoring_dict

backend/tests/test_sap_model_prep.py

@@ -1,15 +1,16 @@
 from backend.Property import Property
 from etl.epc.DataProcessor import DataProcessor
-from backend.app.plan.utils import create_recommendation_scoring_data
+from backend.app.plan.utils import create_recommendation_scoring_data, get_cleaned
 from etl.epc.settings import COLUMNS_TO_MERGE_ON
 from epc_api.client import EpcClient
 import pandas as pd
-import os
 import pytest
-import pickle
+
+from utils.s3 import read_dataframe_from_s3_parquet
 from tqdm import tqdm
 
 
+# import pickle
 # with open("sap_change_dataset.pickle", "rb") as f:
 #     sap_change_dataset = pickle.load(f)
 #
@@ -109,18 +110,14 @@ from tqdm import tqdm
 class TestSapModelPrep:
 
     @pytest.fixture
-    def cleaned(self):
-        with open(
-            os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaned.pickle", "rb"
-        ) as f:
-            return pickle.load(f)
+    def cleaning_data(self):
+        return read_dataframe_from_s3_parquet(
+            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
+        )
 
     @pytest.fixture
-    def cleaning_data(self):
-        with open(
-            os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaning_data.pickle", "rb"
-        ) as f:
-            return pickle.load(f)
+    def cleaned(self):
+        return get_cleaned()
 
     def test_fill_cavity_wall(self, cleaned, cleaning_data):
         """
@@ -395,7 +392,7 @@ class TestSapModelPrep:
             'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0,
             'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0,
             'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307,
-            'walls_thermal_transmittance': 0.45, 'is_cavity_wall': False, 'is_filled_cavity': False,
+            'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, 'is_filled_cavity': False,
             'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False,
             'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False,
             'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none',
@@ -482,7 +479,7 @@ class TestSapModelPrep:
 
         ending_lodgement_date2 = '2020-11-24'
 
-        starting_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2)
+        ending_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2)
 
         recommendation2 = {
             "recommendation_id": 0,

recommendations/tests/test_data/wall_uvalue_test_cases.py

@@ -4,21 +4,21 @@ wall_uvalue_test_cases = [
         "age_band": "A",
         "is_granite_or_whinstone": False,
         "is_sandstone_or_limestone": False,
-        "uvalue": 0.7
+        "uvalue": 1.3
     },
     {
         "clean_description": "Cavity wall, as built, partial insulation",
         "age_band": "F",
         "is_granite_or_whinstone": False,
         "is_sandstone_or_limestone": False,
-        "uvalue": 0.4
+        "uvalue": 0.85
     },
     {
         "clean_description": "Cavity wall, as built, partial insulation",
-        "age_band": "F",
+        "age_band": "G",
         "is_granite_or_whinstone": False,
         "is_sandstone_or_limestone": False,
-        "uvalue": 0.4
+        "uvalue": 0.5375
     },
     {
 

recommendations/tests/test_floor_recommendations.py

@@ -118,6 +118,7 @@ class TestFloorRecommendations:
         input_properties[2].age_band = "A"
         input_properties[2].perimeter = 20
         input_properties[2].wall_type = "solid brick"
+        input_properties[2].floor_type = "suspended"
 
         recommender = FloorRecommendations(
             property_instance=input_properties[2],
@@ -160,6 +161,7 @@ class TestFloorRecommendations:
         input_properties[4].age_band = "B"
         input_properties[4].perimeter = 50
         input_properties[4].wall_type = "solid brick"
+        input_properties[4].floor_type = "solid"
 
         recommender = FloorRecommendations(
             property_instance=input_properties[4],

recommendations/tests/test_wall_recommendations.py

@@ -407,36 +407,8 @@ class TestWallRecommendationsBase:
         wall_recommendations_instance.property.data = {"property-type": "house"}
         assert wall_recommendations_instance.ewi_valid is True
 
-    def test_recommend_without_u_value(self, wall_recommendations_instance):
-        wall_recommendations_instance.property.walls = {
-            "thermal_transmittance": None,
-            "is_solid_brick": False,
-            "is_cavity_wall": False,
-            "insulation_thickness": "none",
-            "clean_description": "Solid brick, as built, no insulation",
-            "is_granite_or_whinstone": False,
-            "is_sandstone_or_limestone": False,
-        }
-        wall_recommendations_instance.property.age_band = "A"
-        with pytest.raises(NotImplementedError):
-            wall_recommendations_instance.recommend()
-
 
 class TestCavityWallRecommensations:
-    data = {
-        'low-energy-fixed-light-count': '', 'address': '123 Fake Street',
-        'floor-height': '', 'construction-age-band': 'England and Wales: 1950-1966',
-        'address3': '', 'property-type': 'House', 'local-authority-label': 'Melton',
-        'county': 'Leicestershire', 'postcode': 'LE14 2QH',
-        'solar-water-heating-flag': 'N', 'constituency': 'E14000909',
-        'number-heated-rooms': '5', 'local-authority': 'E07000133', 'built-form': 'End-Terrace',
-        'address1': '1, 23 fake', 'total-floor-area': '85.0', 'environment-impact-current': '49',
-        'number-habitable-rooms': 3, 'address2': 'Fake', 'posttown': 'IDK',
-        'walls-energy-eff': 'Poor', 'current-energy-rating': 'D',
-        'transaction-type': 'ECO assessment', 'uprn': '999', 'current-energy-efficiency': '57',
-        'lodgement-date': '2019-07-10', 'lmk-key': '999', 'tenure': 'rental (private)', 'floor-level': 'NODATA!',
-        'walls-description': 'Cavity wall, as built, no insulation (assumed)',
-    }
 
     def test_fill_empty_cavity(self):
         input_property = Property(id=1, postcode="F4k3", address1="123 fake street", epc_client=Mock())
@@ -465,10 +437,10 @@ class TestCavityWallRecommensations:
 
         assert recommender.recommendations
         assert recommender.estimated_u_value == 1.5
-        assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.25)
+        assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.37)
         assert np.isclose(recommender.recommendations[0]["cost"], 1000)
 
-        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.26)
+        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.38)
         assert np.isclose(recommender.recommendations[1]["cost"], 1250)
 
     def test_fill_partial_filled_cavity(self):
@@ -498,10 +470,10 @@ class TestCavityWallRecommensations:
 
         assert recommender.recommendations
         assert recommender.estimated_u_value == 1.3
-        assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.56)
+        assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.43)
         assert np.isclose(recommender.recommendations[0]["cost"], 1000)
 
-        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.57)
+        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.45)
         assert np.isclose(recommender.recommendations[1]["cost"], 1250)
 
     def test_system_built_wall(self):