Merge pull request #433 from Hestia-Homes/remote-assessment-api

Remote assessment api

backend/engine/engine.py

@@ -528,7 +528,7 @@ async def model_engine(body: PlanTriggerRequest):
             prepared_epc = EPCRecord(
                 epc_records=epc_records,
                 run_mode="newdata",
-                cleaning_data=cleaning_data
+                cleaning_data=cleaning_data,
             )
 
             input_properties.append(

etl/epc/Record.py

@@ -531,6 +531,8 @@ class EPCRecord:
             or (self.prepared_epc["floor-height"] in DATA_ANOMALY_MATCHES)
             or (self.prepared_epc["number-heated-rooms"] in DATA_ANOMALY_MATCHES)
         ):
+            # TODO - this probably shouldn't live here - but we only need to use this for specific properties
+            # when we meet this condition
             property_dimensions = read_dataframe_from_s3_parquet(
                 bucket_name=DATA_BUCKET,
                 file_key=f"property_dimensions/{self.prepared_epc['local-authority']}.parquet",

etl/epc/tests/test_epcrecord.py

@@ -1,6 +1,5 @@
 import pickle
 import pytest
-from utils.s3 import read_dataframe_from_s3_parquet
 from etl.epc.Record import EPCRecord
 from etl.epc.settings import DATA_ANOMALY_MATCHES
 import random

recommendations/tests/test_data/heating_recommendations_data.py

@@ -823,7 +823,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor',
             'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good',
             'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow',
             'local-authority-label': 'Herefordshire, County of',
@@ -869,7 +869,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
             'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor',
             'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath',
             'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire',
@@ -916,7 +916,7 @@ testing_examples = [
             'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets',
             'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood',
             'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None,
-            'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N',
+            'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N',
             'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse',
             'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire',
             'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900',
@@ -960,7 +960,7 @@ testing_examples = [
             'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets',
             'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)',
             'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None,
-            'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N',
+            'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N',
             'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave',
             'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby',
             'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949',
@@ -1093,7 +1093,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor',
             'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor',
             'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural',
             'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of',
@@ -1228,7 +1228,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
             'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor',
             'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5,
             'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural',
             'address': '16, Woodside', 'local-authority-label': 'Wolverhampton',
             'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON',

recommendations/tests/test_window_recommendations.py

@@ -1,13 +1,27 @@
+import pytest
+import pickle
 from recommendations.WindowsRecommendations import WindowsRecommendations
 from backend.Property import Property
 from recommendations.tests.test_data.materials import materials
 from etl.epc.Record import EPCRecord
-import msgpack
-from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
 
 
 class TestWindowRecommendations:
 
+    @pytest.fixture()
+    def cleaning_data(self):
+        with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f:
+            data = pickle.load(f)
+
+        return data
+
+    @pytest.fixture()
+    def cleaned(self):
+        with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f:
+            df = pickle.load(f)
+
+        return df
+
     def test_fully_single_glazed(self):
         """
         For this property, we expect all windows to be single glazed and should recommend full double glazing
@@ -18,7 +32,10 @@ class TestWindowRecommendations:
             "county": "Wychavon",
             "multi-glaze-proportion": 0,
             "uprn": 0,
-            "windows-energy-eff": "Very Poor"
+            "windows-energy-eff": "Very Poor",
+            "floor-area": 2.5,
+            "number-habitable-rooms": 5,
+            "number-heated-rooms": 5,
         }
         property_1 = Property(
             id=1,
@@ -317,13 +334,7 @@ class TestWindowRecommendations:
 
         assert not recommender8.recommendation
 
-    def test_simulating_outcome_single_glazed(self):
-        # Could move these to fixtures
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-        cleaned = read_from_s3(s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev")
-        cleaned = msgpack.unpackb(cleaned, raw=False)
+    def test_simulating_outcome_single_glazed(self, cleaning_data, cleaned):
 
         epc = {
             'lmk-key': 'f4cf43c90ab3140112a9d1c8cfb21ec1bf73f5a2ca3c75118f289d3447dddf15', 'address1': '3 The Green',