Merge pull request #439 from Hestia-Homes/main

Fixing unit test runner

.github/workflows/unit_tests.yml

@@ -6,33 +6,31 @@ on:
       - main
 
 jobs:
-  build:
-
+  test:
     runs-on: ubuntu-latest
-    # For the moment, we just run with python 3.10
-    #    strategy:
-    #      matrix:
-    #        python-version: [ 3.8, 3.9, 3.10 ]
 
     steps:
-      - uses: actions/checkout@v2
-      #      - name: Set up Python ${{ matrix.python-version }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
         with:
-          #          python-version: ${{ matrix.python-version }}
-          python-version: '3.10'
-      - name: Install dependencies
+          python-version: '3.11'
+
+      - name: Install tox via Makefile
         run: |
-          python -m pip install --upgrade pip
-          pip install -r model_data/requirements/requirements.txt
+          make setup
+
       - name: Set dev AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
-      - name: Run tests with pytest
+
+      - name: Run tests with tox via Makefile
+        env:
+          EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
         run: |
-          pip install -r model_data/requirements/dev.txt
-          pytest
+          make test

backend/engine/engine.py

@@ -528,7 +528,7 @@ async def model_engine(body: PlanTriggerRequest):
             prepared_epc = EPCRecord(
                 epc_records=epc_records,
                 run_mode="newdata",
-                cleaning_data=cleaning_data
+                cleaning_data=cleaning_data,
             )
 
             input_properties.append(

backend/tests/test_property.py

@@ -110,7 +110,7 @@ class TestProperty:
 
         return property_instance
 
-    @pytest.fixture
+    @pytest.fixture()
     def mock_cleaner(self):
         lighting_averages = [
             {'lighting-description': 'good lighting efficiency', 'low-energy-lighting': 99.26666666666667},

backend/tests/test_search_epc.py

@@ -3,11 +3,17 @@ import os
 from backend.SearchEpc import SearchEpc  # Replace with your actual module name
 from dotenv import load_dotenv
 
-load_dotenv(dotenv_path="backend/.env")
-EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+# If this backend/.env file is present, we load it
+if os.path.exists("backend/.env"):
+    load_dotenv("backend/.env")
 
 
 class TestSearchEpcIntegration:
+
+    @pytest.fixture
+    def epc_auth_token(self):
+        return os.getenv("EPC_AUTH_TOKEN")
+
     @pytest.mark.parametrize(
         "address, postcode, uprn, skip_os, lmk_key, n_old_epcs",
         [
@@ -29,7 +35,7 @@ class TestSearchEpcIntegration:
 
         ],
     )
-    def test_find_property(self, address, postcode, uprn, skip_os, lmk_key, n_old_epcs):
+    def test_find_property(self, epc_auth_token, address, postcode, uprn, skip_os, lmk_key, n_old_epcs):
         """
         Integration test for `find_property`, making actual API calls.
         """
@@ -41,7 +47,7 @@ class TestSearchEpcIntegration:
             address1=address,
             postcode=postcode,
             uprn=uprn,
-            auth_token=EPC_AUTH_TOKEN,
+            auth_token=epc_auth_token,
             os_api_key=os_api_key,
         )
 

etl/epc/Record.py

@@ -531,6 +531,8 @@ class EPCRecord:
             or (self.prepared_epc["floor-height"] in DATA_ANOMALY_MATCHES)
             or (self.prepared_epc["number-heated-rooms"] in DATA_ANOMALY_MATCHES)
         ):
+            # TODO - this probably shouldn't live here - but we only need to use this for specific properties
+            # when we meet this condition
             property_dimensions = read_dataframe_from_s3_parquet(
                 bucket_name=DATA_BUCKET,
                 file_key=f"property_dimensions/{self.prepared_epc['local-authority']}.parquet",

etl/epc/tests/test_epcrecord.py

@@ -1,5 +1,5 @@
+import pickle
 import pytest
-from utils.s3 import read_dataframe_from_s3_parquet
 from etl.epc.Record import EPCRecord
 from etl.epc.settings import DATA_ANOMALY_MATCHES
 import random
@@ -9,11 +9,10 @@ class TestEpcRecord:
 
     @pytest.fixture()
     def cleaning_data(self):
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
+        with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f:
+            data = pickle.load(f)
 
-        return cleaning_data
+        return data
 
     @pytest.fixture()
     def epc_records_1(self):
@@ -262,7 +261,7 @@ class TestEpcRecord:
         record.full_sap_epc = []
         record._clean_number_lighting_outlets()
 
-        assert record.prepared_epc["fixed-lighting-outlets-count"] == 8.0
+        assert record.prepared_epc["fixed-lighting-outlets-count"] == 10
 
     def test_clean_count_variables(self, cleaning_data):
         record = EPCRecord(cleaning_data=cleaning_data)

recommendations/tests/test_data/heating_recommendations_data.py

@@ -823,7 +823,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor',
             'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good',
             'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow',
             'local-authority-label': 'Herefordshire, County of',
@@ -869,7 +869,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
             'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor',
             'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath',
             'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire',
@@ -916,7 +916,7 @@ testing_examples = [
             'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets',
             'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood',
             'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None,
-            'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N',
+            'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N',
             'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse',
             'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire',
             'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900',
@@ -960,7 +960,7 @@ testing_examples = [
             'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets',
             'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)',
             'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None,
-            'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N',
+            'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N',
             'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave',
             'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby',
             'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949',
@@ -1093,7 +1093,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor',
             'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor',
             'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5,
             'photo-supply': None,
             'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural',
             'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of',
@@ -1228,7 +1228,7 @@ testing_examples = [
             'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average',
             'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor',
             'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0,
-            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None,
+            'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5,
             'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural',
             'address': '16, Woodside', 'local-authority-label': 'Wolverhampton',
             'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON',

recommendations/tests/test_heating_recommendations.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 import pandas as pd
-import msgpack
-from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
+import pickle
 import pytest
 from backend.Property import Property
 from etl.epc.Record import EPCRecord
@@ -12,23 +11,21 @@ from recommendations.tests.test_data.heating_recommendations_data import testing
 
 class TestHeatingRecommendations:
 
-    @pytest.fixture
+    @pytest.fixture()
     def cleaning_data(self):
-        return read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
+        with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f:
+            data = pickle.load(f)
 
-    @pytest.fixture
+        return data
+
+    @pytest.fixture()
     def cleaned(self):
-        df = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
+        with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f:
+            df = pickle.load(f)
 
-        df = msgpack.unpackb(df, raw=False)
         return df
 
-    @pytest.fixture
+    @pytest.fixture()
     def kwh_client(self):
         client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False)
         # We fix this pricing table for these tests

recommendations/tests/test_window_recommendations.py

@@ -1,13 +1,27 @@
+import pytest
+import pickle
 from recommendations.WindowsRecommendations import WindowsRecommendations
 from backend.Property import Property
 from recommendations.tests.test_data.materials import materials
 from etl.epc.Record import EPCRecord
-import msgpack
-from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
 
 
 class TestWindowRecommendations:
 
+    @pytest.fixture()
+    def cleaning_data(self):
+        with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f:
+            data = pickle.load(f)
+
+        return data
+
+    @pytest.fixture()
+    def cleaned(self):
+        with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f:
+            df = pickle.load(f)
+
+        return df
+
     def test_fully_single_glazed(self):
         """
         For this property, we expect all windows to be single glazed and should recommend full double glazing
@@ -18,7 +32,10 @@ class TestWindowRecommendations:
             "county": "Wychavon",
             "multi-glaze-proportion": 0,
             "uprn": 0,
-            "windows-energy-eff": "Very Poor"
+            "windows-energy-eff": "Very Poor",
+            "floor-area": 2.5,
+            "number-habitable-rooms": 5,
+            "number-heated-rooms": 5,
         }
         property_1 = Property(
             id=1,
@@ -317,13 +334,7 @@ class TestWindowRecommendations:
 
         assert not recommender8.recommendation
 
-    def test_simulating_outcome_single_glazed(self):
-        # Could move these to fixtures
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-        cleaned = read_from_s3(s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev")
-        cleaned = msgpack.unpackb(cleaned, raw=False)
+    def test_simulating_outcome_single_glazed(self, cleaning_data, cleaned):
 
         epc = {
             'lmk-key': 'f4cf43c90ab3140112a9d1c8cfb21ec1bf73f5a2ca3c75118f289d3447dddf15', 'address1': '3 The Green',

tox.ini

@@ -3,6 +3,7 @@ envlist = py311
 skipsdist = True
 
 [testenv]
+passenv = EPC_AUTH_TOKEN
 description = Install dependencies and run tests
 deps =
     -rbackend/engine/requirements.txt