diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 43149791..78f08f3c 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -1,9 +1,11 @@ +from datetime import datetime import pandas as pd import pytest from unittest.mock import Mock from backend.Property import Property from etl.epc_clean.EpcClean import EpcClean from etl.epc.Record import EPCRecord +from etl.bill_savings.KwhData import KwhData # Define some test data mock_epc_response = { @@ -17,12 +19,13 @@ mock_epc_response = { "built-form": "Detached", "inspection-date": "2023-06-01", 'lodgement-datetime': '2023-06-01 20:29:01', + 'lodgement-date': '2023-06-01', "some-other-key": "some-value", "roof-description": "pitched, no insulation", "walls-description": "Walls Description", - "windows-description": "Windows Description", - "mainheat-description": "Main Heating Description", - "hotwater-description": "Hot Water Description", + "windows-description": "Fully double glazed", + "mainheat-description": "Boiler and radiators, mains gas", + "hotwater-description": "From main system", "transaction-type": "rental", "lighting-description": "Good Lighting Efficiency", "energy-consumption-current": "50", @@ -39,7 +42,10 @@ mock_epc_response = { "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", "floor-description": "Floor Description", - "floor-level": "Ground" + "floor-level": "Ground", + "lighting-cost-current": 123, + "heating-cost-current": 800, + "hot-water-cost-current": 200 }, { "lmk-key": 2, @@ -49,12 +55,13 @@ mock_epc_response = { "built-form": "Detached", "inspection-date": "2023-05-01", 'lodgement-datetime': '2023-05-01 20:29:01', + 'lodgement-date': '2023-05-01', "some-other-key": "some-other-value", "roof-description": "Roof Description", "walls-description": "Walls Description", - "windows-description": "Windows Description", - "mainheat-description": "Main Heating Description", - "hotwater-description": "Hot Water Description", + "windows-description": "Fully double glazed", + "mainheat-description": "Boiler and radiators, mains gas", + "hotwater-description": "From main system", "transaction-type": "rental", "lighting-description": "Good Lighting Efficiency", "energy-consumption-current": "50", @@ -71,98 +78,10 @@ mock_epc_response = { "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", "floor-description": "Floor Description", - "floor-level": "Ground" - } - ] -} - -mock_epc_response_dupe = { - 'rows': [ - { - "lmk-key": 1, - "uprn": 1, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-06-01', - 'lodgement-datetime': '2023-06-01 20:29:01', - 'some-other-key': 'some-value', 'roof-description': 'Roof Description', - 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', - 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" - }, - { - "lmk-key": 2, - "uprn": 2, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-05-01', - 'lodgement-datetime': '2023-05-01 20:29:01', - 'some-other-key': 'some-other-value', - 'roof-description': 'Roof Description', 'walls-description': 'Walls Description', - 'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description', - 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" - }, - { - "lmk-key": 3, - "uprn": 3, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-06-01', - 'lodgement-datetime': '2023-06-01 20:29:01', - 'some-other-key': 'duplicate-date', - 'roof-description': 'Roof Description', - 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', - 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" + "floor-level": "Ground", + "lighting-cost-current": 123, + "heating-cost-current": 800, + "hot-water-cost-current": 200 } ] } @@ -170,34 +89,14 @@ mock_epc_response_dupe = { class TestProperty: - @pytest.fixture(autouse=True) - def mock_photo_supply_lookup(self): - return pd.DataFrame( - [ - dict( - tenure="rental (social)", - built_form="Detached", - property_type="House", - construction_age_band="England and Wales: 1967-1975", - is_flat=False, - is_pitched=True, - is_roof_room=False, - floor_area_decile=2, - photo_supply_median=40 - ) - ] - ) - - @pytest.fixture(autouse=True) - def mock_floor_area_decile_thresholds(self): - return pd.DataFrame( - {"floor_area_decile_thresholds": [0, 10, 30, 50]} - ) - @pytest.fixture(autouse=True) def property_instance(self, mock_cleaner): epc_record = EPCRecord() - epc_record.prepared_epc = mock_epc_response["rows"][0] + prepared_epc = mock_epc_response["rows"][0].copy() + # Replace hyphens with underscores + prepared_epc = {k.replace("-", "_"): v for k, v in prepared_epc.items()} + epc_record.prepared_epc = prepared_epc + epc_record.uprn = prepared_epc["uprn"] property_instance = Property(id=1, postcode="AB12CD", address="Test Address", epc_record=epc_record) property_instance.number_of_floors = 2 @@ -206,27 +105,6 @@ class TestProperty: property_instance.floor_height = 2.5 return property_instance - @pytest.fixture(autouse=True) - def property_instance_dupe_data(self): - epc_record = EPCRecord() - epc_record.prepared_epc = mock_epc_response_dupe["rows"][0] - property_instance_dupe_data = Property(id=2, postcode="AB12CD", address="Test Address", epc_record=epc_record) - return property_instance_dupe_data - - # @pytest.fixture - # def mock_epc_client(self): - # mock_epc_client = Mock(spec=EpcClient(auth_token="mocked_auth_token")) - # mock_epc_client.domestic.search.return_value = mock_epc_response.copy() - # mock_epc_client.auth_token = "mocked_auth_token" - # return mock_epc_client - # - # @pytest.fixture - # def mock_epc_client_dupe_data(self): - # mock_epc_client_dupe_data = Mock(spec=EpcClient(auth_token="mocked_auth_token")) - # mock_epc_client_dupe_data.domestic.search.return_value = mock_epc_response_dupe.copy() - # mock_epc_client_dupe_data.auth_token = "mocked_auth_token" - # return mock_epc_client_dupe_data - @pytest.fixture def mock_cleaner(self): lighting_averages = [ @@ -270,15 +148,59 @@ class TestProperty: "is_roof_room": False} ], "walls-description": [walls_data], - "windows-description": [{"original_description": "Windows Description"}], - "mainheat-description": [{"original_description": "Main Heating Description"}], - "hotwater-description": [{"original_description": "Hot Water Description"}], + "windows-description": [ + {'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full', + 'glazing_type': 'double', 'no_data': False} + ], + "mainheat-description": [ + { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, + 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, + 'has_water_source_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, + 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, + 'has_assumed': False, + 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, + "has_electric_heat_pumps": False, + "has_micro-cogeneration": False + } + ], + "hotwater-description": [ + {'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system', + 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, + 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, + 'assumed': False, "appliance": None} + ], "lighting-description": [{"original_description": "Good Lighting Efficiency"}], "floor-description": [ {"original_description": "Floor Description", "is_suspended": True, "another_property_below": False}] } return mock_cleaner + @pytest.fixture + def kwh_client(self): + kwh_client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False) + # We fix this pricing table for these tests + kwh_client.retail_price_comparison = pd.DataFrame( + [ + { + "Date": datetime.today().strftime("%Y-%m-%d"), + 'Average standard variable tariff (Large legacy suppliers)': 1 + } + ] + ) + kwh_client.retail_price_comparison["Date"] = pd.to_datetime(kwh_client.retail_price_comparison["Date"]) + return kwh_client + def test_init(self): epc_record = EPCRecord() epc_record.prepared_epc = {"uprn": 1} @@ -292,13 +214,26 @@ class TestProperty: inst3 = Property(4, "AB12CD", "Test Address", epc_record=epc_record) assert inst3.data == {"uprn": 1} - def test_get_components( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + def test_set_features( + self, property_instance, mock_cleaner, kwh_client, ): - property_instance.get_components( + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + + property_instance.set_features( mock_cleaner.cleaned, - photo_supply_lookup=mock_photo_supply_lookup, - floor_area_decile_thresholds=mock_floor_area_decile_thresholds + kwh_client, + kwh_predictions ) # Verify that the components are set correctly @@ -318,9 +253,32 @@ class TestProperty: "is_sandstone_or_limestone": False, "is_granite_or_whinstone": False, } - assert property_instance.windows == {"original_description": "Windows Description"} - assert property_instance.main_heating == {"original_description": "Main Heating Description"} - assert property_instance.hotwater == {"original_description": "Hot Water Description"} + assert property_instance.windows == { + 'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full', + 'glazing_type': 'double', 'no_data': False + } + assert property_instance.main_heating == { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False, + 'has_micro-cogeneration': False + } + + assert property_instance.hotwater == { + 'original_description': 'From main system', 'heater_type': None, + 'system_type': 'from main system', 'thermostat_characteristics': None, + 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, + 'no_system_present': None, 'assumed': False, 'appliance': None + } assert property_instance.wall_type == "cavity" @@ -330,11 +288,24 @@ class TestProperty: # Verify that ValueError is raised when EpcClean doesn't contain cleaned data with pytest.raises(ValueError, match="Cleaner does not contain cleaned data"): - property_instance.get_components(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame()) + property_instance.set_features(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame()) def test_get_components_no_attributes( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + self, property_instance, mock_cleaner, kwh_client ): + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + # Modify the mock cleaner to have no attributes for a specific description mock_cleaner.cleaned = { "roof-description": [] @@ -351,23 +322,45 @@ class TestProperty: "is_sandstone_or_limestone": False, "is_granite_or_whinstone": False, } - property_instance.floor = { "is_suspended": False, "another_property_below": False, "is_solid": True } + property_instance.main_heating = { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False, + 'has_micro-cogeneration': False + } + property_instance.hotwater = { + 'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system', + 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, + 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, + 'assumed': False, "appliance": None + } # Assert backup cleaning has been applied - property_instance.get_components( - mock_cleaner.cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + property_instance.set_features( + mock_cleaner.cleaned, + kwh_client, + kwh_predictions ) assert property_instance.roof["clean_description"] == "Pitched, no insulation" assert property_instance.roof["is_pitched"] def test_get_components_multiple_attributes( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + self, property_instance, mock_cleaner, kwh_client ): # This shouldn't happen - it would mean a cleaning error property_instance.data["roof-description"] = "Roof Description" @@ -378,13 +371,27 @@ class TestProperty: ] } + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + # Verify that ValueError is raised when multiple attributes are found with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"): - property_instance.get_components(cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds) + property_instance.set_features(cleaned, kwh_client, kwh_predictions) def test_set_spatial(self): epc_record = EPCRecord() epc_record.prepared_epc = mock_epc_response["rows"][0] + epc_record.uprn = mock_epc_response["rows"][0]["uprn"] prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record) spatial1 = pd.DataFrame([{ @@ -418,6 +425,7 @@ class TestProperty: # floor, so we should set floor_level to 0 epc_record = EPCRecord() epc_record.prepared_epc = {'floor-level': '01', 'property-type': 'Flat'} + epc_record.uprn = 1 prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record) prop.floor = { 'original_description': 'Solid, no insulation (assumed)', 'clean_description': 'Solid, no insulation',