From 9f46b23b72623afb217ff30db5ff0d33bf38b8d5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 14 May 2025 15:34:12 +0100 Subject: [PATCH] added testing automation with tox and added new test to handle error case when fetching address from epc registry --- .github/workflows/unit_tests.yml | 5 --- Makefile | 5 +++ README.md | 3 +- asset_list/AssetList.py | 2 +- asset_list/requirements.txt | 2 +- backend/SearchEpc.py | 2 +- backend/engine/requirements.txt | 4 +- backend/tests/test_search_epc.py | 8 +++- etl/spatial/tests/test_borehole_client.py | 40 ------------------- pytest.ini | 4 +- .../dev.txt => test.requirements.txt | 1 + tox.ini | 11 +++++ 12 files changed, 32 insertions(+), 55 deletions(-) create mode 100644 Makefile delete mode 100644 etl/spatial/tests/test_borehole_client.py rename backend/app/requirements/dev.txt => test.requirements.txt (85%) create mode 100644 tox.ini diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 39d285f2..22b13ab2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,8 +36,3 @@ jobs: run: | pip install -r model_data/requirements/dev.txt pytest -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v2 -# with: -# token: ${{ secrets.CODECOV_TOKEN }} -# fail_ci_if_error: true diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e26ab667 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +setup: + pip install tox + +test: + tox \ No newline at end of file diff --git a/README.md b/README.md index df36cfe7..9268ba25 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,5 @@ To run tests in a specific service, e.g. inside of model_data, simply run pytest --cov-config=model_data/.coveragerc --cov=model_data ``` -This will produce the test results and coverage reports \ No newline at end of file +This will produce the test results and coverage reports + diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 529aef3d..4b7a11ec 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -10,7 +10,7 @@ from openai import OpenAI import numpy as np import pandas as pd from tqdm import tqdm -from fuzzywuzzy import process +from thefuzz import process from utils.logger import setup_logger from backend.SearchEpc import SearchEpc from BaseUtility import Definitions diff --git a/asset_list/requirements.txt b/asset_list/requirements.txt index fd43ac64..99943397 100644 --- a/asset_list/requirements.txt +++ b/asset_list/requirements.txt @@ -3,7 +3,7 @@ pandas usaddress pydantic-settings==2.6.0 epc-api-python==1.0.2 -fuzzywuzzy +thefuzz boto3 openpyxl openai diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index e19a776d..0010191a 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -14,7 +14,7 @@ from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from BaseUtility import Definitions from utils.logger import setup_logger from typing import List -from fuzzywuzzy import process +from thefuzz import process from backend.app.utils import sap_to_epc logger = setup_logger() diff --git a/backend/engine/requirements.txt b/backend/engine/requirements.txt index f5e1b5f6..b565e9d3 100644 --- a/backend/engine/requirements.txt +++ b/backend/engine/requirements.txt @@ -10,8 +10,8 @@ boto3==1.35.44 # ML, Data Science usaddress==0.5.11 epc-api-python==1.0.2 -fuzzywuzzy==0.18.0 -python-Levenshtein==0.26.0 +thefuzz +python-Levenshtein>=0.24.0,!=0.26.0 textblob==0.18.0.post0 msgpack==1.1.0 scikit-learn==1.5.2 diff --git a/backend/tests/test_search_epc.py b/backend/tests/test_search_epc.py index 562585ad..d4313f61 100644 --- a/backend/tests/test_search_epc.py +++ b/backend/tests/test_search_epc.py @@ -9,7 +9,7 @@ EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") class TestSearchEpcIntegration: @pytest.mark.parametrize( - "address, postcode, uprn, skip_os, expected_partial_address", + "address, postcode, uprn, skip_os, lmk_key, n_old_epcs", [ # Test case 1: Valid address and postcode, skipping OS # In this case, the property is an individual flat but the uprn associated to the @@ -21,7 +21,11 @@ class TestSearchEpcIntegration: # In this case, the newest EPC, does not have a uprn associated to it. If we did a search by # uprn, we would get an old EPC ("Flat 8, Hainton House", "DN32 9AQ", 10090082018, True, - "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 3), + "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 2), + # Test case 3: When we make a request to the API for this property, we get back results for + # flats 1, 2 and 3. We have some logic to handle the response so that we get back flat 1 + ("Flat 1, 1 Tottenham Street, London", "W1T 2AE", 5167411, True, + "3e6414d7f15f4cf7a69dc20c469bcf043d31a49239b183f1bd0c0e1aafa23c93", 0), ], ) diff --git a/etl/spatial/tests/test_borehole_client.py b/etl/spatial/tests/test_borehole_client.py deleted file mode 100644 index 38bf4495..00000000 --- a/etl/spatial/tests/test_borehole_client.py +++ /dev/null @@ -1,40 +0,0 @@ -import pytest -from etl.spatial.BoreholeClient import BoreholeClient - - -@pytest.fixture -def mock_borehole_data(monkeypatch): - # Define the mock data to be returned by the read() method - mock_data = [ - { - 'X': 464343.0, - 'Y': 415553.0 - }, - { - 'X': 464341.0, - 'Y': 415556.0 - }, - # Add more mock data entries here... - ] - - # Monkeypatch the read() method to return the mock data - def mock_read(self): - return mock_data - - # Apply the monkeypatch to the BoreholeClient class - monkeypatch.setattr(BoreholeClient, 'read', mock_read) - - -@pytest.mark.usefixtures('mock_borehole_data') -def test_distance_between_bng_coords(): - # Create an instance of BoreholeClient - borehole_client = BoreholeClient("path/to/dbf/file") - - # Test the distance calculation - distance_m, distance_km = borehole_client.distance_between_bng_coords( - x1_bng=123456, y1_bng=789012, x2_bng=464343.0, y2_bng=415553.0 - ) - - # Perform assertions to verify the results - assert distance_m == 505643.71987596166 - assert distance_km == 505.64371987596166 diff --git a/pytest.ini b/pytest.ini index b2453c82..84c686b1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] pythonpath = . -addopts = --cov-report term-missing --cov=etl --cov=recommendations --cov=backend -testpaths = etl/*/tests recommendations/tests backend/tests +addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial +testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests diff --git a/backend/app/requirements/dev.txt b/test.requirements.txt similarity index 85% rename from backend/app/requirements/dev.txt rename to test.requirements.txt index a466954c..d31371a6 100644 --- a/backend/app/requirements/dev.txt +++ b/test.requirements.txt @@ -2,3 +2,4 @@ pytest mock pytest-cov pytest-mock +dotenv \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..d9384f58 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py311 +skipsdist = True + +[testenv] +description = Install dependencies and run tests +deps = + -rbackend/engine/requirements.txt + -rtest.requirements.txt +commands = pytest +