diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 22b13ab2..5428fe89 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -6,33 +6,31 @@ on: - main jobs: - build: - + test: runs-on: ubuntu-latest - # For the moment, we just run with python 3.10 - # strategy: - # matrix: - # python-version: [ 3.8, 3.9, 3.10 ] steps: - - uses: actions/checkout@v2 - # - name: Set up Python ${{ matrix.python-version }} - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 with: - # python-version: ${{ matrix.python-version }} - python-version: '3.10' - - name: Install dependencies + python-version: '3.11' + + - name: Install tox via Makefile run: | - python -m pip install --upgrade pip - pip install -r model_data/requirements/requirements.txt + make setup + - name: Set dev AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-2 - - name: Run tests with pytest + + - name: Run tests with tox via Makefile + env: + EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }} run: | - pip install -r model_data/requirements/dev.txt - pytest + make test diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 408d044e..58c3dc8e 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -528,7 +528,7 @@ async def model_engine(body: PlanTriggerRequest): prepared_epc = EPCRecord( epc_records=epc_records, run_mode="newdata", - cleaning_data=cleaning_data + cleaning_data=cleaning_data, ) input_properties.append( diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 7f7cc140..776c1491 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -110,7 +110,7 @@ class TestProperty: return property_instance - @pytest.fixture + @pytest.fixture() def mock_cleaner(self): lighting_averages = [ {'lighting-description': 'good lighting efficiency', 'low-energy-lighting': 99.26666666666667}, diff --git a/backend/tests/test_search_epc.py b/backend/tests/test_search_epc.py index d4313f61..9bb7c39a 100644 --- a/backend/tests/test_search_epc.py +++ b/backend/tests/test_search_epc.py @@ -3,11 +3,17 @@ import os from backend.SearchEpc import SearchEpc # Replace with your actual module name from dotenv import load_dotenv -load_dotenv(dotenv_path="backend/.env") -EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") +# If this backend/.env file is present, we load it +if os.path.exists("backend/.env"): + load_dotenv("backend/.env") class TestSearchEpcIntegration: + + @pytest.fixture + def epc_auth_token(self): + return os.getenv("EPC_AUTH_TOKEN") + @pytest.mark.parametrize( "address, postcode, uprn, skip_os, lmk_key, n_old_epcs", [ @@ -29,7 +35,7 @@ class TestSearchEpcIntegration: ], ) - def test_find_property(self, address, postcode, uprn, skip_os, lmk_key, n_old_epcs): + def test_find_property(self, epc_auth_token, address, postcode, uprn, skip_os, lmk_key, n_old_epcs): """ Integration test for `find_property`, making actual API calls. """ @@ -41,7 +47,7 @@ class TestSearchEpcIntegration: address1=address, postcode=postcode, uprn=uprn, - auth_token=EPC_AUTH_TOKEN, + auth_token=epc_auth_token, os_api_key=os_api_key, ) diff --git a/etl/epc/Record.py b/etl/epc/Record.py index 1ed33567..b8950757 100644 --- a/etl/epc/Record.py +++ b/etl/epc/Record.py @@ -531,6 +531,8 @@ class EPCRecord: or (self.prepared_epc["floor-height"] in DATA_ANOMALY_MATCHES) or (self.prepared_epc["number-heated-rooms"] in DATA_ANOMALY_MATCHES) ): + # TODO - this probably shouldn't live here - but we only need to use this for specific properties + # when we meet this condition property_dimensions = read_dataframe_from_s3_parquet( bucket_name=DATA_BUCKET, file_key=f"property_dimensions/{self.prepared_epc['local-authority']}.parquet", diff --git a/etl/epc/tests/test_epcrecord.py b/etl/epc/tests/test_epcrecord.py index 37a00cd4..feb39c8e 100644 --- a/etl/epc/tests/test_epcrecord.py +++ b/etl/epc/tests/test_epcrecord.py @@ -1,5 +1,5 @@ +import pickle import pytest -from utils.s3 import read_dataframe_from_s3_parquet from etl.epc.Record import EPCRecord from etl.epc.settings import DATA_ANOMALY_MATCHES import random @@ -9,11 +9,10 @@ class TestEpcRecord: @pytest.fixture() def cleaning_data(self): - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) + with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f: + data = pickle.load(f) - return cleaning_data + return data @pytest.fixture() def epc_records_1(self): @@ -262,7 +261,7 @@ class TestEpcRecord: record.full_sap_epc = [] record._clean_number_lighting_outlets() - assert record.prepared_epc["fixed-lighting-outlets-count"] == 8.0 + assert record.prepared_epc["fixed-lighting-outlets-count"] == 10 def test_clean_count_variables(self, cleaning_data): record = EPCRecord(cleaning_data=cleaning_data) diff --git a/recommendations/tests/test_data/cleaned.pkl b/recommendations/tests/test_data/cleaned.pkl new file mode 100644 index 00000000..c18c8b08 Binary files /dev/null and b/recommendations/tests/test_data/cleaned.pkl differ diff --git a/recommendations/tests/test_data/cleaning_data.pkl b/recommendations/tests/test_data/cleaning_data.pkl new file mode 100644 index 00000000..b5a4ac20 Binary files /dev/null and b/recommendations/tests/test_data/cleaning_data.pkl differ diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index a7f4121e..671220bc 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -823,7 +823,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', 'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow', 'local-authority-label': 'Herefordshire, County of', @@ -869,7 +869,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath', 'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire', @@ -916,7 +916,7 @@ testing_examples = [ 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets', 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, - 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse', 'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire', 'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900', @@ -960,7 +960,7 @@ testing_examples = [ 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets', 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, - 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave', 'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby', 'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949', @@ -1093,7 +1093,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', 'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of', @@ -1228,7 +1228,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', 'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', 'address': '16, Woodside', 'local-authority-label': 'Wolverhampton', 'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON', diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index b1ac4d18..039801e1 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -1,7 +1,6 @@ from datetime import datetime import pandas as pd -import msgpack -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 +import pickle import pytest from backend.Property import Property from etl.epc.Record import EPCRecord @@ -12,23 +11,21 @@ from recommendations.tests.test_data.heating_recommendations_data import testing class TestHeatingRecommendations: - @pytest.fixture + @pytest.fixture() def cleaning_data(self): - return read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) + with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f: + data = pickle.load(f) - @pytest.fixture + return data + + @pytest.fixture() def cleaned(self): - df = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) + with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f: + df = pickle.load(f) - df = msgpack.unpackb(df, raw=False) return df - @pytest.fixture + @pytest.fixture() def kwh_client(self): client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False) # We fix this pricing table for these tests diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 1c32d2bc..51a3118e 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -1,13 +1,27 @@ +import pytest +import pickle from recommendations.WindowsRecommendations import WindowsRecommendations from backend.Property import Property from recommendations.tests.test_data.materials import materials from etl.epc.Record import EPCRecord -import msgpack -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 class TestWindowRecommendations: + @pytest.fixture() + def cleaning_data(self): + with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f: + data = pickle.load(f) + + return data + + @pytest.fixture() + def cleaned(self): + with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f: + df = pickle.load(f) + + return df + def test_fully_single_glazed(self): """ For this property, we expect all windows to be single glazed and should recommend full double glazing @@ -18,7 +32,10 @@ class TestWindowRecommendations: "county": "Wychavon", "multi-glaze-proportion": 0, "uprn": 0, - "windows-energy-eff": "Very Poor" + "windows-energy-eff": "Very Poor", + "floor-area": 2.5, + "number-habitable-rooms": 5, + "number-heated-rooms": 5, } property_1 = Property( id=1, @@ -317,13 +334,7 @@ class TestWindowRecommendations: assert not recommender8.recommendation - def test_simulating_outcome_single_glazed(self): - # Could move these to fixtures - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - cleaned = read_from_s3(s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev") - cleaned = msgpack.unpackb(cleaned, raw=False) + def test_simulating_outcome_single_glazed(self, cleaning_data, cleaned): epc = { 'lmk-key': 'f4cf43c90ab3140112a9d1c8cfb21ec1bf73f5a2ca3c75118f289d3447dddf15', 'address1': '3 The Green', diff --git a/tox.ini b/tox.ini index d9384f58..19a4ad9a 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = py311 skipsdist = True [testenv] +passenv = EPC_AUTH_TOKEN description = Install dependencies and run tests deps = -rbackend/engine/requirements.txt