added testing automation with tox and added new test to handle error case when fetching address from epc registry

.github/workflows/unit_tests.yml

@@ -36,8 +36,3 @@ jobs:
         run: |
           pip install -r model_data/requirements/dev.txt
           pytest
-#      - name: Upload coverage to Codecov
-#        uses: codecov/codecov-action@v2
-#        with:
-#          token: ${{ secrets.CODECOV_TOKEN }}
-#          fail_ci_if_error: true

Makefile

@@ -0,0 +1,5 @@
+setup:
+	pip install tox
+
+test:
+	tox

README.md

@@ -37,4 +37,5 @@ To run tests in a specific service, e.g. inside of model_data, simply run
 pytest --cov-config=model_data/.coveragerc --cov=model_data
 ```
 
-This will produce the test results and coverage reports
+This will produce the test results and coverage reports
+

asset_list/AssetList.py

@@ -10,7 +10,7 @@ from openai import OpenAI
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
-from fuzzywuzzy import process
+from thefuzz import process
 from utils.logger import setup_logger
 from backend.SearchEpc import SearchEpc
 from BaseUtility import Definitions

asset_list/requirements.txt

@@ -3,7 +3,7 @@ pandas
 usaddress
 pydantic-settings==2.6.0
 epc-api-python==1.0.2
-fuzzywuzzy
+thefuzz
 boto3
 openpyxl
 openai

backend/SearchEpc.py

@@ -14,7 +14,7 @@ from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
 from BaseUtility import Definitions
 from utils.logger import setup_logger
 from typing import List
-from fuzzywuzzy import process
+from thefuzz import process
 from backend.app.utils import sap_to_epc
 
 logger = setup_logger()

backend/engine/requirements.txt

@@ -10,8 +10,8 @@ boto3==1.35.44
 # ML, Data Science
 usaddress==0.5.11
 epc-api-python==1.0.2
-fuzzywuzzy==0.18.0
+thefuzz
-python-Levenshtein==0.26.0
+python-Levenshtein>=0.24.0,!=0.26.0
 textblob==0.18.0.post0
 msgpack==1.1.0
 scikit-learn==1.5.2

backend/tests/test_search_epc.py

@@ -9,7 +9,7 @@ EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
 
 class TestSearchEpcIntegration:
     @pytest.mark.parametrize(
-        "address, postcode, uprn, skip_os, expected_partial_address",
+        "address, postcode, uprn, skip_os, lmk_key, n_old_epcs",
         [
             # Test case 1: Valid address and postcode, skipping OS
             # In this case, the property is an individual flat but the uprn associated to the
@@ -21,7 +21,11 @@ class TestSearchEpcIntegration:
             # In this case, the newest EPC, does not have a uprn associated to it. If we did a search by
             # uprn, we would get an old EPC
             ("Flat 8, Hainton House", "DN32 9AQ", 10090082018, True,
-             "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 3),
+             "bd1149a20a73397184f07a9955f872424826e70f4870c058d71be887766ee1f8", 2),
+            # Test case 3: When we make a request to the API for this property, we get back results for
+            # flats 1, 2 and 3. We have some logic to handle the response so that we get back flat 1
+            ("Flat 1, 1 Tottenham Street, London", "W1T 2AE", 5167411, True,
+             "3e6414d7f15f4cf7a69dc20c469bcf043d31a49239b183f1bd0c0e1aafa23c93", 0),
 
         ],
     )

etl/spatial/tests/test_borehole_client.py

@@ -1,40 +0,0 @@
-import pytest
-from etl.spatial.BoreholeClient import BoreholeClient
-
-
-@pytest.fixture
-def mock_borehole_data(monkeypatch):
-    # Define the mock data to be returned by the read() method
-    mock_data = [
-        {
-            'X': 464343.0,
-            'Y': 415553.0
-        },
-        {
-            'X': 464341.0,
-            'Y': 415556.0
-        },
-        # Add more mock data entries here...
-    ]
-
-    # Monkeypatch the read() method to return the mock data
-    def mock_read(self):
-        return mock_data
-
-    # Apply the monkeypatch to the BoreholeClient class
-    monkeypatch.setattr(BoreholeClient, 'read', mock_read)
-
-
-@pytest.mark.usefixtures('mock_borehole_data')
-def test_distance_between_bng_coords():
-    # Create an instance of BoreholeClient
-    borehole_client = BoreholeClient("path/to/dbf/file")
-
-    # Test the distance calculation
-    distance_m, distance_km = borehole_client.distance_between_bng_coords(
-        x1_bng=123456, y1_bng=789012, x2_bng=464343.0, y2_bng=415553.0
-    )
-
-    # Perform assertions to verify the results
-    assert distance_m == 505643.71987596166
-    assert distance_km == 505.64371987596166

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 pythonpath = .
-addopts = --cov-report term-missing --cov=etl --cov=recommendations --cov=backend
+addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial
-testpaths = etl/*/tests recommendations/tests backend/tests
+testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests

test.requirements.txt

@@ -2,3 +2,4 @@ pytest
 mock
 pytest-cov
 pytest-mock
+dotenv

tox.ini

@@ -0,0 +1,11 @@
+[tox]
+envlist = py311
+skipsdist = True
+
+[testenv]
+description = Install dependencies and run tests
+deps =
+    -rbackend/engine/requirements.txt
+    -rtest.requirements.txt
+commands = pytest
+