From 2838beaa8ad043bae6f7e983ea38cff36d8b0637 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 15 May 2025 17:03:15 +0100 Subject: [PATCH 1/2] Added missing floor heights --- backend/engine/engine.py | 2 +- etl/epc/Record.py | 2 ++ .../tests/test_data/heating_recommendations_data.py | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 408d044e..58c3dc8e 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -528,7 +528,7 @@ async def model_engine(body: PlanTriggerRequest): prepared_epc = EPCRecord( epc_records=epc_records, run_mode="newdata", - cleaning_data=cleaning_data + cleaning_data=cleaning_data, ) input_properties.append( diff --git a/etl/epc/Record.py b/etl/epc/Record.py index 1ed33567..b8950757 100644 --- a/etl/epc/Record.py +++ b/etl/epc/Record.py @@ -531,6 +531,8 @@ class EPCRecord: or (self.prepared_epc["floor-height"] in DATA_ANOMALY_MATCHES) or (self.prepared_epc["number-heated-rooms"] in DATA_ANOMALY_MATCHES) ): + # TODO - this probably shouldn't live here - but we only need to use this for specific properties + # when we meet this condition property_dimensions = read_dataframe_from_s3_parquet( bucket_name=DATA_BUCKET, file_key=f"property_dimensions/{self.prepared_epc['local-authority']}.parquet", diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index a7f4121e..671220bc 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -823,7 +823,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', 'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow', 'local-authority-label': 'Herefordshire, County of', @@ -869,7 +869,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.4, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath', 'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire', @@ -916,7 +916,7 @@ testing_examples = [ 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets', 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, - 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse', 'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire', 'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900', @@ -960,7 +960,7 @@ testing_examples = [ 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets', 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, - 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave', 'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby', 'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949', @@ -1093,7 +1093,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', 'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': None, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of', @@ -1228,7 +1228,7 @@ testing_examples = [ 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', 'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, - 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', 'address': '16, Woodside', 'local-authority-label': 'Wolverhampton', 'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON', From 1b15f66f48ba3909cb78e87bc532d81708073e87 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 15 May 2025 17:12:19 +0100 Subject: [PATCH 2/2] removing online dependency from tests --- etl/epc/tests/test_epcrecord.py | 1 - .../tests/test_window_recommendations.py | 31 +++++++++++++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/etl/epc/tests/test_epcrecord.py b/etl/epc/tests/test_epcrecord.py index 39e1ee60..feb39c8e 100644 --- a/etl/epc/tests/test_epcrecord.py +++ b/etl/epc/tests/test_epcrecord.py @@ -1,6 +1,5 @@ import pickle import pytest -from utils.s3 import read_dataframe_from_s3_parquet from etl.epc.Record import EPCRecord from etl.epc.settings import DATA_ANOMALY_MATCHES import random diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 1c32d2bc..51a3118e 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -1,13 +1,27 @@ +import pytest +import pickle from recommendations.WindowsRecommendations import WindowsRecommendations from backend.Property import Property from recommendations.tests.test_data.materials import materials from etl.epc.Record import EPCRecord -import msgpack -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 class TestWindowRecommendations: + @pytest.fixture() + def cleaning_data(self): + with open("recommendations/tests/test_data/cleaning_data.pkl", "rb") as f: + data = pickle.load(f) + + return data + + @pytest.fixture() + def cleaned(self): + with open("recommendations/tests/test_data/cleaned.pkl", "rb") as f: + df = pickle.load(f) + + return df + def test_fully_single_glazed(self): """ For this property, we expect all windows to be single glazed and should recommend full double glazing @@ -18,7 +32,10 @@ class TestWindowRecommendations: "county": "Wychavon", "multi-glaze-proportion": 0, "uprn": 0, - "windows-energy-eff": "Very Poor" + "windows-energy-eff": "Very Poor", + "floor-area": 2.5, + "number-habitable-rooms": 5, + "number-heated-rooms": 5, } property_1 = Property( id=1, @@ -317,13 +334,7 @@ class TestWindowRecommendations: assert not recommender8.recommendation - def test_simulating_outcome_single_glazed(self): - # Could move these to fixtures - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - cleaned = read_from_s3(s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev") - cleaned = msgpack.unpackb(cleaned, raw=False) + def test_simulating_outcome_single_glazed(self, cleaning_data, cleaned): epc = { 'lmk-key': 'f4cf43c90ab3140112a9d1c8cfb21ec1bf73f5a2ca3c75118f289d3447dddf15', 'address1': '3 The Green',