From 482ddb1e98799701ebd342ddc065a78967291f3f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 12:29:02 +1100 Subject: [PATCH 01/75] fixing property tests, added cavity wall fill test --- backend/app/plan/utils.py | 1 - backend/requirements/dev.txt | 4 + backend/tests/test_property.py | 60 +++-- backend/tests/test_sap_model_prep.py | 331 +++++++++++++++++++++++++++ 4 files changed, 382 insertions(+), 14 deletions(-) create mode 100644 backend/requirements/dev.txt diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 2e5f204a..0565ab94 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -3,7 +3,6 @@ from backend.Property import Property from collections import defaultdict from utils.s3 import read_from_s3 -from recommendations.config import UPGRADES_MAP from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value from backend.app.db.utils import row2dict diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt new file mode 100644 index 00000000..a466954c --- /dev/null +++ b/backend/requirements/dev.txt @@ -0,0 +1,4 @@ +pytest +mock +pytest-cov +pytest-mock diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 76a299fc..d7028bc6 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -13,6 +13,7 @@ mock_epc_response = { "number-habitable-rooms": 5, "property-type": "House", "inspection-date": "2023-06-01", + 'lodgement-datetime': '2023-06-01 20:29:01', "some-other-key": "some-value", "roof-description": "Roof Description", "walls-description": "Walls Description", @@ -33,7 +34,8 @@ mock_epc_response = { "mains-gas-flag": "Y", "floor-height": 2.5, "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975" + "construction-age-band": "England and Wales: 1967-1975", + "floor-description": "Floor Description" }, { "lmk-key": 2, @@ -41,6 +43,7 @@ mock_epc_response = { "number-habitable-rooms": 5, "property-type": "House", "inspection-date": "2023-05-01", + 'lodgement-datetime': '2023-05-01 20:29:01', "some-other-key": "some-other-value", "roof-description": "Roof Description", "walls-description": "Walls Description", @@ -61,7 +64,8 @@ mock_epc_response = { "mains-gas-flag": "Y", "floor-height": 2.5, "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975" + "construction-age-band": "England and Wales: 1967-1975", + "floor-description": "Floor Description" } ] } @@ -73,7 +77,9 @@ mock_epc_response_dupe = { "uprn": 1, "number-habitable-rooms": 5, "property-type": "House", - 'inspection-date': '2023-06-01', 'some-other-key': 'some-value', 'roof-description': 'Roof Description', + 'inspection-date': '2023-06-01', + 'lodgement-datetime': '2023-06-01 20:29:01', + 'some-other-key': 'some-value', 'roof-description': 'Roof Description', 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', "transaction-type": "rental", @@ -90,14 +96,17 @@ mock_epc_response_dupe = { "mains-gas-flag": "Y", "floor-height": 2.5, "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975" + "construction-age-band": "England and Wales: 1967-1975", + "floor-description": "Floor Description" }, { "lmk-key": 2, "uprn": 2, "number-habitable-rooms": 5, "property-type": "House", - 'inspection-date': '2023-05-01', 'some-other-key': 'some-other-value', + 'inspection-date': '2023-05-01', + 'lodgement-datetime': '2023-05-01 20:29:01', + 'some-other-key': 'some-other-value', 'roof-description': 'Roof Description', 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', @@ -115,14 +124,17 @@ mock_epc_response_dupe = { "mains-gas-flag": "Y", "floor-height": 2.5, "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975" + "construction-age-band": "England and Wales: 1967-1975", + "floor-description": "Floor Description" }, { "lmk-key": 3, "uprn": 3, "number-habitable-rooms": 5, "property-type": "House", - 'inspection-date': '2023-06-01', 'some-other-key': 'duplicate-date', + 'inspection-date': '2023-06-01', + 'lodgement-datetime': '2023-06-01 20:29:01', + 'some-other-key': 'duplicate-date', 'roof-description': 'Roof Description', 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', @@ -140,7 +152,8 @@ mock_epc_response_dupe = { "mains-gas-flag": "Y", "floor-height": 2.5, "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975" + "construction-age-band": "England and Wales: 1967-1975", + "floor-description": "Floor Description" } ] } @@ -187,7 +200,8 @@ class TestProperty: {"mainheat-description": "Main Heating Description"}, {"hotwater-description": "Hot Water Description"}, {"lighting-description": "Good Lighting Efficiency"}, - {"low-energy-lighting": 0} + {"low-energy-lighting": 0}, + {"floor-description": "Floor Description"} ], lighting_averages=lighting_averages ) @@ -212,7 +226,8 @@ class TestProperty: "windows-description": [{"original_description": "Windows Description"}], "mainheat-description": [{"original_description": "Main Heating Description"}], "hotwater-description": [{"original_description": "Hot Water Description"}], - "lighting-description": [{"original_description": "Good Lighting Efficiency"}] + "lighting-description": [{"original_description": "Good Lighting Efficiency"}], + "floor-description": [{"original_description": "Floor Description", "is_suspended": True}] } return mock_cleaner @@ -288,14 +303,33 @@ class TestProperty: "roof-description": [] } property_instance.search_address_epc() + property_instance.data["roof-description"] = "Pitched, no insulation" + property_instance.walls = { + "original_description": "Walls Description", + "is_cavity_wall": True, + "is_solid_brick": False, + "is_timber_frame": False, + "is_system_built": False, + "is_park_home": False, + "is_cob": False, + "is_sandstone_or_limestone": False, + "is_granite_or_whinstone": False, + } - # Verify that ValueError is raised when no attributes are found - with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"): - property_instance.get_components(mock_cleaner.cleaned) + property_instance.floor = { + "is_suspended": False + } + + # Assert backup cleaning has been applied + property_instance.get_components(mock_cleaner.cleaned) + + assert property_instance.roof["clean_description"] == "Pitched, no insulation" + assert property_instance.roof["is_pitched"] def test_get_components_multiple_attributes(self, property_instance, mock_cleaner): # This shouldn't happen - it would mean a cleaning error property_instance.search_address_epc() + property_instance.data["roof-description"] = "Roof Description" cleaned = { "roof-description": [ {"original_description": "Roof Description"}, diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index e69de29b..1f08da00 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -0,0 +1,331 @@ +from backend.Property import Property +from etl.epc.DataProcessor import DataProcessor +from backend.app.plan.utils import create_recommendation_scoring_data +from etl.epc.settings import COLUMNS_TO_MERGE_ON +from epc_api.client import EpcClient +import pandas as pd +import os +import pytest +import pickle +from tqdm import tqdm + + +# import pickle +# +# with open("sap_change_dataset.pickle", "rb") as f: +# sap_change_dataset = pickle.load(f) +# +# search_from = sap_change_dataset[ +# sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"] +# ] +# search_from = search_from[ +# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & +# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & +# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & +# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & +# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) +# ] +# +# # Find a record where the only difference is cavity wall getting filled +# ending_cols = [c for c in search_from.columns if "_ENDING" in c] +# +# ignore = [ +# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", +# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +# ] +# +# ending_cols = [c for c in ending_cols if c not in ignore] +# +# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): +# +# same = True +# starting_cols = [] +# for c in ending_cols: +# +# starting_col = c.replace("_ENDING", "") +# if starting_col not in search_from.columns: +# starting_col = c.replace("_ENDING", "_STARTING") +# if starting_col not in search_from.columns: +# raise Exception("something went wrong") +# +# starting_cols.append(starting_col) +# +# # We want them to be different +# if c == "walls_thermal_transmittance_ENDING": +# if row[c] == row[starting_col]: +# same = False +# break +# else: +# continue +# +# # We now check if the starting and ending values are the same +# if row[c] != row[starting_col]: +# same = False +# break +# +# if same: +# raise Exception("We found one!") +# +# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] +# +# import pandas as pd +# +# start = row[starting_cols] +# start.index = [c.replace("_STARTING", "") for c in start.index] +# end = row[ending_cols] +# end.index = [c.replace("_ENDING", "") for c in end.index] +# start["type"] = "starting" +# end["type"] = "ending" +# +# compare = pd.concat([start, end], axis=1) +# +# ending_lmk = "1481856809222016121510000597528546" +# starting_lmk = "1481856849902016092320290148762028" +# +# EPC_AUTH_TOKEN = "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=" +# +# client = EpcClient(auth_token=EPC_AUTH_TOKEN) +# result = client.domestic.search(params={"address": "26, Vicarage Lane", "postcode": "NG32 1SP"}) +# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] + + +# with open("cleaned.pickle", "wb") as f: +# pickle.dump(cleaned, f) + + +# with open("cleaning_data.pickle", "wb") as f: +# pickle.dump(cleaning_data, f) + + +class TestSapModelPrep: + + @pytest.fixture + def cleaned(self): + with open( + os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaned.pickle", "rb" + ) as f: + return pickle.load(f) + + @pytest.fixture + def cleaning_data(self): + with open( + os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaning_data.pickle", "rb" + ) as f: + return pickle.load(f) + + def test_fill_cavity_wall(self, cleaned, cleaning_data): + """ + We ensure that the process that prepares the data in the engine code results in the same data as + the model is trained on + """ + + # This is an actual starting EPC + starting_epc = { + 'low-energy-fixed-light-count': '', 'address': '26, Vicarage Lane, Eaton', + 'uprn-source': 'Address Matched', 'floor-height': '2.39', 'heating-cost-potential': '942', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '97', + 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'D', + 'mainheat-energy-eff': 'Average', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Average', + 'environment-impact-potential': '53', + 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '1475', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Melton', + 'fixed-lighting-outlets-count': '', 'energy-tariff': 'Single', + 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96', 'county': 'Leicestershire', + 'postcode': 'NG32 1SP', 'solar-water-heating-flag': 'Y', 'constituency': 'E14000909', + 'co2-emissions-potential': '5.7', 'number-heated-rooms': '7', + 'floor-description': 'Suspended, no insulation (assumed)', + 'energy-consumption-potential': '177', 'local-authority': 'E07000133', 'built-form': 'Detached', + 'number-open-fireplaces': '1', 'windows-description': 'Fully double glazed', + 'glazed-area': 'Normal', 'inspection-date': '2016-09-22', 'mains-gas-flag': 'N', + 'co2-emiss-curr-per-floor-area': '87', 'address1': '26, Vicarage Lane', + 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', + 'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Very Poor', + 'total-floor-area': '116.0', 'building-reference-number': '4940047478', + 'environment-impact-current': '29', 'co2-emissions-current': '10.0', + 'roof-description': 'Pitched, limited insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'number-habitable-rooms': '7', 'address2': 'Eaton', 'hot-water-env-eff': 'Good', + 'posttown': 'GRANTHAM', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'oil (not community)', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 31% of fixed outlets', + 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Poor', 'photo-supply': '', + 'lighting-cost-potential': '69', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', + 'main-heating-controls': '2106', 'lodgement-datetime': '2016-09-23 20:29:01', + 'flat-top-storey': '', 'current-energy-rating': 'F', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', 'walls-env-eff': 'Poor', + 'transaction-type': 'marketed sale', 'uprn': '100030534042', 'current-energy-efficiency': '34', + 'energy-consumption-current': '343', 'mainheat-description': 'Boiler and radiators, oil', + 'lighting-cost-current': '117', 'lodgement-date': '2016-09-23', 'extension-count': '2', + 'mainheatc-env-eff': 'Good', 'lmk-key': '1481856849902016092320290148762028', + 'wind-turbine-count': '0', 'tenure': 'owner-occupied', 'floor-level': 'NODATA!', + 'potential-energy-efficiency': '64', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '31', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', + 'hotwater-description': 'From main system, plus solar' + } + + # This is the training data as we prepare it in the engine + # This is an actual record from the training data + row = { + 'UPRN': '100030534042', 'RDSAP_CHANGE': 12, 'HEAT_DEMAND_CHANGE': -72, + 'CARBON_CHANGE': -2.0999999999999996, 'SAP_STARTING': 34, 'SAP_ENDING': 46, 'HEAT_DEMAND_STARTING': 343, + 'HEAT_DEMAND_ENDING': 271, 'CARBON_STARTING': 10.0, 'CARBON_ENDING': 7.9, 'PROPERTY_TYPE': 'House', + 'BUILT_FORM': 'Detached', 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 7.0, + 'NUMBER_HEATED_ROOMS': 7.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 21.0, + 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1967-1975', 'TRANSACTION_TYPE_STARTING': 'marketed sale', + 'MECHANICAL_VENTILATION_STARTING': 'natural', + 'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_STARTING': 'Single', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'Y', + 'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002', + 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, 'LOW_ENERGY_LIGHTING_STARTING': 31.0, + 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, 'EXTENSION_COUNT_STARTING': 2.0, + 'TOTAL_FLOOR_AREA_STARTING': 116.0, 'FLOOR_HEIGHT_STARTING': 2.39, + 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural', + 'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_ENDING': 'Single', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'Y', 'PHOTO_SUPPLY_ENDING': 0.0, + 'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002', + 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, 'LOW_ENERGY_LIGHTING_ENDING': 31.0, + 'NUMBER_OPEN_FIREPLACES_ENDING': 1.0, 'EXTENSION_COUNT_ENDING': 2.0, 'TOTAL_FLOOR_AREA_ENDING': 116.0, + 'FLOOR_HEIGHT_ENDING': 2.41, 'DAYS_TO_STARTING': 784, 'DAYS_TO_ENDING': 867, + 'walls_thermal_transmittance': 1.5, 'is_cavity_wall': True, 'is_filled_cavity': False, + 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, + 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.7, + 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'none', + 'external_insulation_ENDING': False, 'internal_insulation_ENDING': False, + 'floor_thermal_transmittance': 0.64, 'is_to_unheated_space': False, 'is_to_external_air': False, + 'is_suspended': True, 'is_solid': False, 'another_property_below': False, + 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.64, + 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 1.5, 'is_pitched': True, + 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'has_dwelling_above': False, 'roof_insulation_thickness': 'below average', + 'roof_thermal_transmittance_ENDING': 1.5, 'roof_insulation_thickness_ENDING': 'below average', + 'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown', + 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown', + 'extra_features': 'plus solar', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', + 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'Unknown', + 'system_type_ENDING': 'from main system', 'thermostat_characteristics_ENDING': 'Unknown', + 'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown', + 'hotwater_tariff_type_ENDING': 'Unknown', 'extra_features_ENDING': 'plus solar', + 'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown', + 'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': True, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, + 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, + 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, + 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, + 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, + 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, + 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, + 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, + 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, + 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, + 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, + 'has_electric_ENDING': False, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False, + 'has_coal_ENDING': False, 'has_oil_ENDING': True, 'has_wood_pellets_ENDING': False, + 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, + 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, + 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, + 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat', + 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', + 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, + 'auxiliary_systems': 'Unknown', 'trvs': 'trvs', 'rate_control': 'Unknown', + 'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown', + 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', + 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, + 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'trvs', 'rate_control_ENDING': 'Unknown', + 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'oil', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'fuel_type_ENDING': 'oil', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, + 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', + 'estimated_perimeter_STARTING': 44.77882152472145, 'estimated_perimeter_ENDING': 44.77882152472145 + } + + home = Property( + id=0, + postcode=starting_epc["postcode"], + address1=starting_epc["address1"], + epc_client=EpcClient(auth_token="notoken"), + data=starting_epc + ) + home.get_components(cleaned) + + data_processor = DataProcessor(None, newdata=True) + data_processor.insert_data(pd.DataFrame([home.get_model_data()])) + + data_processor.pre_process() + + starting_epc_data = data_processor.get_component_features(suffix="_STARTING") + ending_epc_data = data_processor.get_component_features(suffix="_ENDING") + fixed_data = data_processor.get_fixed_features() + + ending_lodgement_date = '2016-12-15' + + ending_epc_data["DAYS_TO_ENDING"] = data_processor.calculate_days_to(ending_lodgement_date) + + recommendation = { + "recommendation_id": 0, + "new_u_value": 0.7, + "type": "wall_insulation" + } + + test_record = create_recommendation_scoring_data( + property=home, + recommendation=recommendation, + starting_epc_data=starting_epc_data, + ending_epc_data=ending_epc_data, + fixed_data=fixed_data, + ) + test_record = pd.DataFrame([test_record]) + + # Test the final cleaning: + test_record = DataProcessor.apply_averages_cleaning( + data_to_clean=test_record, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + ).drop(columns=["LOCAL_AUTHORITY"]) + + test_record = DataProcessor.clean_missings_after_description_process( + test_record, [ + c for c in test_record.columns if + ("thermal_transmittance" in c) or ("insulation_thickness" in c) + ] + ) + + # Test that the data has been set up correctly + + # Things to fix: + # [] Filled cavity should have an average insulation thickness in the cleaned data + + for c in test_record.columns: + if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: + continue + + if c == "FLOOR_HEIGHT_ENDING": + assert (row[c] - test_record[c].values[0]) <= 0.020001 + continue + + if c == "walls_insulation_thickness_ENDING": + print("Add back in the checks") + continue + assert row[c] == "average" + assert test_record[c] == "above average" + + assert test_record[c].values[0] == row[c] + + def test_solid_wall_insulation(self): + pass From 13490b18caeef6c075cdae543c821af2825dee7f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 14:13:42 +1100 Subject: [PATCH 02/75] Added tests for solid walls --- backend/tests/test_sap_model_prep.py | 373 +++++++++++++++++++++------ etl/epc/property_change_app.py | 6 +- 2 files changed, 292 insertions(+), 87 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 1f08da00..f621391d 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -9,93 +9,100 @@ import pytest import pickle from tqdm import tqdm +with open("sap_change_dataset.pickle", "rb") as f: + sap_change_dataset = pickle.load(f) -# import pickle -# -# with open("sap_change_dataset.pickle", "rb") as f: -# sap_change_dataset = pickle.load(f) -# -# search_from = sap_change_dataset[ -# sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"] -# ] -# search_from = search_from[ -# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & -# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & -# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & -# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & -# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) -# ] -# -# # Find a record where the only difference is cavity wall getting filled -# ending_cols = [c for c in search_from.columns if "_ENDING" in c] -# -# ignore = [ -# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", -# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -# ] -# -# ending_cols = [c for c in ending_cols if c not in ignore] -# -# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): -# -# same = True -# starting_cols = [] -# for c in ending_cols: -# -# starting_col = c.replace("_ENDING", "") -# if starting_col not in search_from.columns: -# starting_col = c.replace("_ENDING", "_STARTING") -# if starting_col not in search_from.columns: -# raise Exception("something went wrong") -# -# starting_cols.append(starting_col) -# -# # We want them to be different -# if c == "walls_thermal_transmittance_ENDING": -# if row[c] == row[starting_col]: -# same = False -# break -# else: -# continue -# -# # We now check if the starting and ending values are the same -# if row[c] != row[starting_col]: -# same = False -# break -# -# if same: -# raise Exception("We found one!") -# -# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] -# -# import pandas as pd -# -# start = row[starting_cols] -# start.index = [c.replace("_STARTING", "") for c in start.index] -# end = row[ending_cols] -# end.index = [c.replace("_ENDING", "") for c in end.index] -# start["type"] = "starting" -# end["type"] = "ending" -# -# compare = pd.concat([start, end], axis=1) -# -# ending_lmk = "1481856809222016121510000597528546" -# starting_lmk = "1481856849902016092320290148762028" -# -# EPC_AUTH_TOKEN = "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=" -# -# client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "26, Vicarage Lane", "postcode": "NG32 1SP"}) -# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] +search_from = sap_change_dataset[ + (sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]) & + (sap_change_dataset["is_solid_brick"]) + ] +search_from = search_from[ + (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & + (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & + (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & + (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & + (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) + ] + +# Find a record where the only difference is cavity wall getting filled +ending_cols = [c for c in search_from.columns if "_ENDING" in c] + +ignore = [ + "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", + "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +] + +ending_cols = [c for c in ending_cols if c not in ignore] + +for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): + + same = True + starting_cols = [] + for c in ending_cols: + + starting_col = c.replace("_ENDING", "") + if starting_col not in search_from.columns: + starting_col = c.replace("_ENDING", "_STARTING") + if starting_col not in search_from.columns: + raise Exception("something went wrong") + + starting_cols.append(starting_col) + + # We want them to be different + if c == "walls_thermal_transmittance_ENDING": + if row[c] == row[starting_col]: + same = False + break + else: + continue + + # We want them to be different + if c == "walls_insulation_thickness_ENDING": + if row[c] == row[starting_col]: + same = False + break + else: + continue + + # We now check if the starting and ending values are the same + if row[c] != row[starting_col]: + same = False + break + + if same: + raise Exception("We found one!") + + fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] + + import pandas as pd + + start = row[starting_cols] + start.index = [c.replace("_STARTING", "") for c in start.index] + end = row[ending_cols] + end.index = [c.replace("_ENDING", "") for c in end.index] + start["type"] = "starting" + end["type"] = "ending" + + compare = pd.concat([start, end], axis=1) + +ending_lmk = "d0fc64d6b80db04c32998c9b846dd04c8f0b486231a11e4c062020b35af1312d" +starting_lmk = "b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d" + +client = EpcClient(auth_token=EPC_AUTH_TOKEN) +result = client.domestic.search(params={"address": "FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY", "postcode": "E16 1BD"}) +starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] -# with open("cleaned.pickle", "wb") as f: -# pickle.dump(cleaned, f) +# with open( +# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaned.pickle", "rb" +# ) as f: +# cleaned = pickle.load(f) - -# with open("cleaning_data.pickle", "wb") as f: -# pickle.dump(cleaning_data, f) +# with open( +# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaning_data.pickle", "rb" +# ) as f: +# cleaning_data = pickle.load(f) class TestSapModelPrep: @@ -327,5 +334,201 @@ class TestSapModelPrep: assert test_record[c].values[0] == row[c] - def test_solid_wall_insulation(self): - pass + def test_solid_wall_insulation(self, cleaned, cleaning_data): + + starting_epc2 = { + 'low-energy-fixed-light-count': '2', 'address': 'FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY', + 'uprn-source': 'Energy Assessor', 'floor-height': '3.64', 'heating-cost-potential': '465', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '185', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Poor', + 'environment-impact-potential': '51', 'glazed-type': 'double glazing installed during or after 2002', + 'heating-cost-current': '1223', 'address3': '3 WESTERN GATEWAY', + 'mainheatcont-description': 'Programmer and appliance thermostats', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Newham', 'fixed-lighting-outlets-count': '12', + 'energy-tariff': 'off-peak 7 hour', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '342', + 'county': '', 'postcode': 'E16 1BD', 'solar-water-heating-flag': 'N', 'constituency': 'E14001032', + 'co2-emissions-potential': '3.6', 'number-heated-rooms': '2', 'floor-description': '(other premises below)', + 'energy-consumption-potential': '307', 'local-authority': 'E09000025', 'built-form': 'Mid-Terrace', + 'number-open-fireplaces': '0', 'windows-description': 'Partial double glazing', 'glazed-area': 'Normal', + 'inspection-date': '2020-10-14', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '66', + 'address1': 'FLAT 12', 'heat-loss-corridor': 'heated corridor', 'flat-storey-count': '', + 'constituency-label': 'West Ham', 'roof-energy-eff': 'N/A', 'total-floor-area': '70.0', + 'building-reference-number': '10000539740', 'environment-impact-current': '42', + 'co2-emissions-current': '4.6', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '2', 'address2': 'WAREHOUSE W', 'hot-water-env-eff': 'Poor', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Poor', + 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 17% of fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '67', + 'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '61', 'main-heating-controls': '', + 'lodgement-datetime': '2020-10-14 00:00:00', 'flat-top-storey': 'N', 'current-energy-rating': 'F', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'marketed sale', + 'uprn': '10012839482', 'current-energy-efficiency': '33', 'energy-consumption-current': '393', + 'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '110', + 'lodgement-date': '2020-10-14', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': 'b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d', 'wind-turbine-count': '0', + 'tenure': 'Owner-occupied', 'floor-level': '01', 'potential-energy-efficiency': '71', + 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '17', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'Electric immersion, standard tariff' + } + + row2 = { + 'UPRN': '10012839482', 'RDSAP_CHANGE': 8, 'HEAT_DEMAND_CHANGE': -59, + 'CARBON_CHANGE': -0.5999999999999996, 'SAP_STARTING': 33, 'SAP_ENDING': 41, 'HEAT_DEMAND_STARTING': 393, + 'HEAT_DEMAND_ENDING': 334, 'CARBON_STARTING': 4.6, 'CARBON_ENDING': 4.0, 'PROPERTY_TYPE': 'Flat', + 'BUILT_FORM': 'Mid-Terrace', 'CONSTITUENCY': 'E14001032', 'NUMBER_HABITABLE_ROOMS': 2.0, + 'NUMBER_HEATED_ROOMS': 2.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 12.0, + 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1996-2002', 'TRANSACTION_TYPE_STARTING': 'marketed sale', + 'MECHANICAL_VENTILATION_STARTING': 'natural', 'SECONDHEAT_DESCRIPTION_STARTING': 'None', + 'ENERGY_TARIFF_STARTING': 'off-peak 7 hour', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', + 'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002', + 'MULTI_GLAZE_PROPORTION_STARTING': 61.0, 'LOW_ENERGY_LIGHTING_STARTING': 17.0, + 'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, 'EXTENSION_COUNT_STARTING': 0.0, + 'TOTAL_FLOOR_AREA_STARTING': 70.0, 'FLOOR_HEIGHT_STARTING': 3.64, + 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural', + 'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'off-peak 7 hour', + 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0, + 'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002', + 'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0, + 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0, + 'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307, + 'walls_thermal_transmittance': 0.45, 'is_cavity_wall': False, 'is_filled_cavity': False, + 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, + 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.21, + 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average', + 'external_insulation_ENDING': False, 'internal_insulation_ENDING': False, + 'floor_thermal_transmittance': 0.0, 'is_to_unheated_space': False, 'is_to_external_air': False, + 'is_suspended': False, 'is_solid': False, 'another_property_below': True, + 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.0, + 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.0, 'is_pitched': False, + 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'has_dwelling_above': True, 'roof_insulation_thickness': 'none', + 'roof_thermal_transmittance_ENDING': 0.0, 'roof_insulation_thickness_ENDING': 'none', + 'heater_type': 'electric immersion', 'system_type': 'Unknown', 'thermostat_characteristics': 'Unknown', + 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'standard tariff', + 'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', + 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'electric immersion', + 'system_type_ENDING': 'Unknown', 'thermostat_characteristics_ENDING': 'Unknown', + 'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown', + 'hotwater_tariff_type_ENDING': 'standard tariff', 'extra_features_ENDING': 'Unknown', + 'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown', + 'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': False, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': True, + 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': False, + 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, + 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, + 'has_boiler_ENDING': False, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': True, + 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, + 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, + 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, + 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, + 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, + 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, + 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, + 'has_electric_ENDING': True, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False, + 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, + 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, + 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, + 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, + 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'appliance thermostats', + 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', + 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, + 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown', + 'thermostatic_control_ENDING': 'appliance thermostats', 'charging_system_ENDING': 'Unknown', + 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', + 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, + 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown', + 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'electricity', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'fuel_type_ENDING': 'electricity', 'main-fuel_tariff_type_ENDING': 'Unknown', + 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, + 'complex_fuel_type_ENDING': 'Unknown', 'estimated_perimeter_STARTING': 35.4964786985977, + 'estimated_perimeter_ENDING': 35.4964786985977 + } + + home2 = Property( + id=0, + postcode=starting_epc2["postcode"], + address1=starting_epc2["address1"], + epc_client=EpcClient(auth_token="notoken"), + data=starting_epc2 + ) + home2.get_components(cleaned) + + data_processor2 = DataProcessor(None, newdata=True) + data_processor2.insert_data(pd.DataFrame([home2.get_model_data()])) + + data_processor2.pre_process() + + starting_epc_data2 = data_processor2.get_component_features(suffix="_STARTING") + ending_epc_data2 = data_processor2.get_component_features(suffix="_ENDING") + fixed_data2 = data_processor2.get_fixed_features() + + ending_lodgement_date2 = '2020-11-24' + + starting_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) + + recommendation2 = { + "recommendation_id": 0, + "new_u_value": 0.21, + "type": "wall_insulation" + } + + test_record2 = create_recommendation_scoring_data( + property=home2, + recommendation=recommendation2, + starting_epc_data=starting_epc_data2, + ending_epc_data=ending_epc_data2, + fixed_data=fixed_data2, + ) + test_record2 = pd.DataFrame([test_record2]) + + # Test the final cleaning: + test_record2 = DataProcessor.apply_averages_cleaning( + data_to_clean=test_record2, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + ).drop(columns=["LOCAL_AUTHORITY"]) + + test_record2 = DataProcessor.clean_missings_after_description_process( + test_record2, [ + c for c in test_record2.columns if + ("thermal_transmittance" in c) or ("insulation_thickness" in c) + ] + ) + + for c in test_record2.columns: + if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: + continue + + if c == "FLOOR_HEIGHT_ENDING": + assert (row2[c] - test_record2[c].values[0]) <= 0.020001 + continue + + if c == "walls_insulation_thickness_ENDING": + assert row2[c] == "average" + assert test_record2[c].values[0] == "above average" + continue + + if c == "CONSTRUCTION_AGE_BAND": + # For this, we have different values in the original data + assert row2[c] == "England and Wales: 1996-2002" + assert test_record2[c].values[0] == "England and Wales: 1900-1929" + continue + + assert test_record2[c].values[0] == row2[c] diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index d7dce61c..067d7161 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -499,10 +499,12 @@ def app(): # Add some temporal features - we look at the days from the standard starting point in time # for the starting and ending date so all records are from a fixed point data_by_urpn_df["DAYS_TO_STARTING"] = DataProcessor.calculate_days_to( - data_by_urpn_df["LODGEMENT_DATE_STARTING"]) + data_by_urpn_df["LODGEMENT_DATE_STARTING"] + ) data_by_urpn_df["DAYS_TO_ENDING"] = DataProcessor.calculate_days_to( - data_by_urpn_df["LODGEMENT_DATE_ENDING"]) + data_by_urpn_df["LODGEMENT_DATE_ENDING"] + ) data_by_urpn_df = data_by_urpn_df.drop(columns=["LODGEMENT_DATE_STARTING", "LODGEMENT_DATE_ENDING"]) From 11a69f4e21603c0116e31f2478f647608c2ef885 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 14:14:11 +1100 Subject: [PATCH 03/75] removed temp code --- backend/tests/test_sap_model_prep.py | 165 ++++++++++++++------------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index f621391d..4f36dbe8 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -9,89 +9,90 @@ import pytest import pickle from tqdm import tqdm -with open("sap_change_dataset.pickle", "rb") as f: - sap_change_dataset = pickle.load(f) -search_from = sap_change_dataset[ - (sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]) & - (sap_change_dataset["is_solid_brick"]) - ] -search_from = search_from[ - (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & - (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & - (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & - (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & - (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) - ] - -# Find a record where the only difference is cavity wall getting filled -ending_cols = [c for c in search_from.columns if "_ENDING" in c] - -ignore = [ - "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", - "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -] - -ending_cols = [c for c in ending_cols if c not in ignore] - -for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): - - same = True - starting_cols = [] - for c in ending_cols: - - starting_col = c.replace("_ENDING", "") - if starting_col not in search_from.columns: - starting_col = c.replace("_ENDING", "_STARTING") - if starting_col not in search_from.columns: - raise Exception("something went wrong") - - starting_cols.append(starting_col) - - # We want them to be different - if c == "walls_thermal_transmittance_ENDING": - if row[c] == row[starting_col]: - same = False - break - else: - continue - - # We want them to be different - if c == "walls_insulation_thickness_ENDING": - if row[c] == row[starting_col]: - same = False - break - else: - continue - - # We now check if the starting and ending values are the same - if row[c] != row[starting_col]: - same = False - break - - if same: - raise Exception("We found one!") - - fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] - - import pandas as pd - - start = row[starting_cols] - start.index = [c.replace("_STARTING", "") for c in start.index] - end = row[ending_cols] - end.index = [c.replace("_ENDING", "") for c in end.index] - start["type"] = "starting" - end["type"] = "ending" - - compare = pd.concat([start, end], axis=1) - -ending_lmk = "d0fc64d6b80db04c32998c9b846dd04c8f0b486231a11e4c062020b35af1312d" -starting_lmk = "b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d" - -client = EpcClient(auth_token=EPC_AUTH_TOKEN) -result = client.domestic.search(params={"address": "FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY", "postcode": "E16 1BD"}) -starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] +# with open("sap_change_dataset.pickle", "rb") as f: +# sap_change_dataset = pickle.load(f) +# +# search_from = sap_change_dataset[ +# (sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]) & +# (sap_change_dataset["is_solid_brick"]) +# ] +# search_from = search_from[ +# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & +# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & +# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & +# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & +# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) +# ] +# +# # Find a record where the only difference is cavity wall getting filled +# ending_cols = [c for c in search_from.columns if "_ENDING" in c] +# +# ignore = [ +# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", +# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +# ] +# +# ending_cols = [c for c in ending_cols if c not in ignore] +# +# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): +# +# same = True +# starting_cols = [] +# for c in ending_cols: +# +# starting_col = c.replace("_ENDING", "") +# if starting_col not in search_from.columns: +# starting_col = c.replace("_ENDING", "_STARTING") +# if starting_col not in search_from.columns: +# raise Exception("something went wrong") +# +# starting_cols.append(starting_col) +# +# # We want them to be different +# if c == "walls_thermal_transmittance_ENDING": +# if row[c] == row[starting_col]: +# same = False +# break +# else: +# continue +# +# # We want them to be different +# if c == "walls_insulation_thickness_ENDING": +# if row[c] == row[starting_col]: +# same = False +# break +# else: +# continue +# +# # We now check if the starting and ending values are the same +# if row[c] != row[starting_col]: +# same = False +# break +# +# if same: +# raise Exception("We found one!") +# +# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] +# +# import pandas as pd +# +# start = row[starting_cols] +# start.index = [c.replace("_STARTING", "") for c in start.index] +# end = row[ending_cols] +# end.index = [c.replace("_ENDING", "") for c in end.index] +# start["type"] = "starting" +# end["type"] = "ending" +# +# compare = pd.concat([start, end], axis=1) +# +# ending_lmk = "d0fc64d6b80db04c32998c9b846dd04c8f0b486231a11e4c062020b35af1312d" +# starting_lmk = "b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d" +# +# client = EpcClient(auth_token=EPC_AUTH_TOKEN) +# result = client.domestic.search(params={"address": "FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY", "postcode": "E16 1BD"}) +# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] # with open( From bb0f51486890c7af805a798f0a530972f4c48608 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 14:20:26 +1100 Subject: [PATCH 04/75] make floor with property below 0 u-value --- etl/epc_clean/epc_attributes/FloorAttributes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 6631b4d5..245a91bc 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -107,4 +107,8 @@ class FloorAttributes(Definitions): else: result['insulation_thickness'] = None + if result["another_property_below"]: + result["thermal_transmittance"] = 0 + result["thermal_transmittance_unit"] = 'w/m-¦k' + return result From 2ead4906be968ba238221fae03b7072e0bfeccdd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 14:23:50 +1100 Subject: [PATCH 05/75] Making roof with premises above 0 u-value --- etl/epc_clean/epc_attributes/RoofAttributes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 9e400235..ed2b4d07 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -138,4 +138,8 @@ class RoofAttributes(Definitions): if "insulation_thickness" not in result: result['insulation_thickness'] = None + if result["has_dwelling_above"]: + result["thermal_transmittance"] = 0 + result["thermal_transmittance_unit"] = 'w/m-¦k' + return result From 40a6d2041ee4567bce4a46c8742a1143825ecb92 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 15:38:17 +1100 Subject: [PATCH 06/75] Added potential variables to dataset --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- etl/epc/DataProcessor.py | 17 ++++++++++ etl/epc/property_change_app.py | 60 ++++++++++++++++++++++++++++++---- etl/epc/settings.py | 41 ++++++++++++++++++++--- 5 files changed, 108 insertions(+), 14 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 3b05c6ac..ca0e1cd9 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index e9c84c3c..c500c095 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -569,3 +569,20 @@ class DataProcessor: df[col] = df[col].fillna("Unknown") return df + + @staticmethod + def clean_efficiency_variables(df): + missings = pd.isnull(df).sum() + missings = missings[missings >= 1] + + if len(missings) == 0: + return df + + # Make sure they are all efficiency columns + if any(~missings.index.str.contains("ENERGY_EFF")): + raise ValueError("Non efficiency columns are missing") + + for m in missings.index: + df[m] = df[m].fillna("NO_RATING") + + return df diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 067d7161..12f6f0c0 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -12,6 +12,10 @@ from etl.epc.settings import ( HEAT_DEMAND_RESPONSE, COLUMNS_TO_MERGE_ON, CARBON_RESPONSE, + CORE_COMPONENT_FEATURES, + EFFICIENCY_FEATURES, + POTENTIAL_COLUMNS, + MINIMUM_FLOOR_HEIGHT ) from etl.epc.DataProcessor import DataProcessor from utils.s3 import save_dataframe_to_s3_parquet, read_from_s3 @@ -363,6 +367,25 @@ def make_uvalues(df): return df +def compare_records(earliest_record: pd.Series, latest_record: pd.Series, columns: list): + """ + For a list of columns, check if the earliest and latest record are the same + If they are the same, we indicate this, because we have example of SAP scores changing + without any feature changes + :param earliest_record: pd.Series + :param latest_record: pd.Series + :param columns: list of columns to compare + :return: boolean indicating whether or not all features are the same + """ + + all_equal = True + for col in columns: + if earliest_record[col] != latest_record[col]: + return False + if all_equal: + return True + + def app(): # Get all the files in the directory @@ -376,6 +399,8 @@ def app(): dataset = [] cleaning_dataset = [] + # Keep track of the number of all equals + all_equal_count = 0 for directory in tqdm(directories): @@ -422,7 +447,9 @@ def app(): # We include the lodgement date here as we probably need to factor time into the # model, since EPC standards and rigour have changed over time variable_data = property_data[ - COMPONENT_FEATURES + ["LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE, CARBON_RESPONSE] + COMPONENT_FEATURES + EFFICIENCY_FEATURES + POTENTIAL_COLUMNS + [ + "LODGEMENT_DATE", RDSAP_RESPONSE, HEAT_DEMAND_RESPONSE, CARBON_RESPONSE + ] ] # Note: we look at changes between subsequent EPCS, however we could look at other permutations @@ -439,6 +466,8 @@ def app(): # Check if the sap gets better or worse gets_better = earliest_record[RDSAP_RESPONSE] <= latest_record[RDSAP_RESPONSE] + component_variables = COMPONENT_FEATURES + EFFICIENCY_FEATURES + if gets_better: starting_sap = earliest_record[RDSAP_RESPONSE] starting_heat_demand = earliest_record[HEAT_DEMAND_RESPONSE] @@ -452,8 +481,8 @@ def app(): heat_demand_change = latest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand carbon_change = latest_record[CARBON_RESPONSE] - starting_carbon - starting_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING") - ending_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING") + starting_record = earliest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_STARTING") + ending_record = latest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_ENDING") else: starting_sap = latest_record[RDSAP_RESPONSE] starting_heat_demand = latest_record[HEAT_DEMAND_RESPONSE] @@ -467,12 +496,23 @@ def app(): heat_demand_change = earliest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand carbon_change = earliest_record[CARBON_RESPONSE] - starting_carbon - starting_record = latest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_STARTING") - ending_record = earliest_record[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].add_suffix("_ENDING") + starting_record = latest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_STARTING") + ending_record = earliest_record[component_variables + ["LODGEMENT_DATE"]].add_suffix("_ENDING") if rdsap_change == 0: continue + all_equal = compare_records( + earliest_record=earliest_record, + latest_record=latest_record, + columns=CORE_COMPONENT_FEATURES + ) + + if all_equal: + # Keep track of this for the moment + all_equal_count += 1 + continue + features = pd.concat([starting_record, ending_record]) property_model_data.append( @@ -487,6 +527,10 @@ def app(): "HEAT_DEMAND_ENDING": ending_heat_demand, "CARBON_STARTING": starting_carbon, "CARBON_ENDING": ending_carbon, + "POTENTIAL_ENERGY_EFFICIENCY": earliest_record["POTENTIAL_ENERGY_EFFICIENCY"], + "ENVIRONMENT_IMPACT_POTENTIAL": earliest_record["ENVIRONMENT_IMPACT_POTENTIAL"], + "ENERGY_CONSUMPTION_POTENTIAL": earliest_record["ENERGY_CONSUMPTION_POTENTIAL"], + "CO2_EMISSIONS_POTENTIAL": earliest_record["CO2_EMISSIONS_POTENTIAL"], **fixed_data, **features.to_dict(), } @@ -496,8 +540,6 @@ def app(): data_by_urpn_df = pd.DataFrame(data_by_urpn) - # Add some temporal features - we look at the days from the standard starting point in time - # for the starting and ending date so all records are from a fixed point data_by_urpn_df["DAYS_TO_STARTING"] = DataProcessor.calculate_days_to( data_by_urpn_df["LODGEMENT_DATE_STARTING"] ) @@ -508,6 +550,8 @@ def app(): data_by_urpn_df = data_by_urpn_df.drop(columns=["LODGEMENT_DATE_STARTING", "LODGEMENT_DATE_ENDING"]) + data_by_urpn_df = DataProcessor.clean_efficiency_variables(data_by_urpn_df) + # We look for key building fabric features that have changed from one EPC to the next. # if, for example, we see that a home has gone from being a cavity wall to a solid wall, we # remove this record, as it indicates that the quality of the EPC conducted in the first instance @@ -541,6 +585,8 @@ def app(): cleaning_averages["LOCAL_AUTHORITY"] = df["LOCAL_AUTHORITY"].values[0] cleaning_dataset.append(cleaning_averages) + print("Final all equal count: %s" % str(all_equal_count)) + # Store cleaning dataset in s3 as a parquet file cleaning_dataset = pd.concat(cleaning_dataset) save_dataframe_to_s3_parquet( diff --git a/etl/epc/settings.py b/etl/epc/settings.py index fb8e464d..93b8929b 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -85,8 +85,7 @@ FIXED_FEATURES = [ "FIXED_LIGHTING_OUTLETS_COUNT", ] -COMPONENT_FEATURES = [ - "TRANSACTION_TYPE", +CORE_COMPONENT_FEATURES = [ "WALLS_DESCRIPTION", "FLOOR_DESCRIPTION", "LIGHTING_DESCRIPTION", @@ -96,21 +95,49 @@ COMPONENT_FEATURES = [ "MAIN_FUEL", "MECHANICAL_VENTILATION", "SECONDHEAT_DESCRIPTION", - "ENERGY_TARIFF", # Not sure if this is relevant - "SOLAR_WATER_HEATING_FLAG", - "PHOTO_SUPPLY", "WINDOWS_DESCRIPTION", "GLAZED_TYPE", "MULTI_GLAZE_PROPORTION", "LOW_ENERGY_LIGHTING", "NUMBER_OPEN_FIREPLACES", "MAINHEATCONT_DESCRIPTION", + "SOLAR_WATER_HEATING_FLAG", + "PHOTO_SUPPLY", +] + +EFFICIENCY_FEATURES = [ + 'HOT_WATER_ENERGY_EFF', + 'FLOOR_ENERGY_EFF', + 'WINDOWS_ENERGY_EFF', + 'WALLS_ENERGY_EFF', + 'SHEATING_ENERGY_EFF', + 'ROOF_ENERGY_EFF', + 'MAINHEAT_ENERGY_EFF', + 'MAINHEATC_ENERGY_EFF', + 'LIGHTING_ENERGY_EFF' +] + +COMPONENT_FEATURES = CORE_COMPONENT_FEATURES + [ + "TRANSACTION_TYPE", + "ENERGY_TARIFF", # Not sure if this is relevant "EXTENSION_COUNT", "TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", # 'GLAZED_AREA', # May not need this since we have MULTI_GLAZE_PROPORTION ] +POTENTIAL_COLUMNS = [ + 'POTENTIAL_ENERGY_RATING', + 'POTENTIAL_ENERGY_EFFICIENCY', + 'ENVIRONMENT_IMPACT_POTENTIAL', + 'ENERGY_CONSUMPTION_POTENTIAL', + 'CO2_EMISSIONS_POTENTIAL', + # We don't include cost features for the moment + # 'LIGHTING_COST_POTENTIAL', + # 'HEATING_COST_POTENTIAL', + # 'HOT_WATER_COST_POTENTIAL' +] + # For these fields, we take the latest value if we have multiple values # Since more recent EPCs have been conducted with more rigour, we assume that the latest value is # the most accurate @@ -253,3 +280,7 @@ ENDING_SUFFIX_COMPONENT_COLS = [ 'rate_control', 'glazing_type', 'fuel_type', 'main-fuel_tariff_type', 'is_community', 'no_individual_heating_or_community_network', 'complex_fuel_type', 'estimated_perimeter' ] + +# We found that without performing any filtering, the bottom 0.5% of homes had a floor height of 1.65m. We'll therefore +# filter out any homes with a floor height below this +MINIMUM_FLOOR_HEIGHT = 1.65 From 3659e45fc20c4e0eb38b4879b626e08755e29195 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 16:06:27 +1100 Subject: [PATCH 07/75] implemented new insulation thickness values --- etl/epc/DataProcessor.py | 11 +++++++++++ etl/epc/property_change_app.py | 12 ++++++++++++ etl/epc_clean/epc_attributes/WallAttributes.py | 9 +++++++++ .../test_data/test_floor_attributes_cases.py | 16 ++++++++-------- .../test_data/test_roof_attributes_cases.py | 14 ++++++++------ .../test_data/test_wall_attributes_cases.py | 10 +++++----- etl/epc_clean/tests/test_roof_attributes.py | 4 ++-- 7 files changed, 55 insertions(+), 21 deletions(-) diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index c500c095..cb3de9f4 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -572,6 +572,17 @@ class DataProcessor: @staticmethod def clean_efficiency_variables(df): + + """ + These is scope to clean this by the model per corresponding description. + E.g. for WALLS_ENG_EFF we could look at the mode efficiency rating by description and + fill in the missing values with this. + When looking at this initially, there are a large volume of records with missing energy efficiency + values and therefore a simpler approach was taken just to test including these variables + :param df: + :return: + """ + missings = pd.isnull(df).sum() missings = missings[missings >= 1] diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 12f6f0c0..300c0417 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -258,6 +258,13 @@ def make_uvalues(df): # Roof # ~~~~~~~~~~~~~~~~~~ + if x["has_dwelling_above"]: + if x["roof_thermal_transmittance"] != 0: + raise ValueError("Should have 0 u-value for roof") + + if x["roof_thermal_transmittance_ENDING"] != 0: + raise ValueError("Should have 0 u-value for roof") + starting_roof_uvalue = x["roof_thermal_transmittance"] if pd.isnull(starting_roof_uvalue): starting_roof_uvalue = get_roof_u_value( @@ -301,6 +308,11 @@ def make_uvalues(df): wall_type = get_wall_type(**x) if x["another_property_below"]: + if x["floor_thermal_transmittance"] != 0: + raise ValueError("Should have 0 u-value for floor") + + if x["floor_thermal_transmittance_ENDING"] != 0: + raise ValueError("Should have 0 u-value for floor") starting_floor_uvalue, ending_floor_uvalue = 0, 0 else: starting_floor_uvalue = x["floor_thermal_transmittance"] diff --git a/etl/epc_clean/epc_attributes/WallAttributes.py b/etl/epc_clean/epc_attributes/WallAttributes.py index 03fe6d67..40a5d5db 100644 --- a/etl/epc_clean/epc_attributes/WallAttributes.py +++ b/etl/epc_clean/epc_attributes/WallAttributes.py @@ -133,4 +133,13 @@ class WallAttributes(Definitions): result['external_insulation'] = 'external insulation' in description result['internal_insulation'] = 'internal insulation' in description + if result["is_filled_cavity"]: + # If it has a filled cavity + internal/external insulation, it's deemed to have above average insulation + if result["external_insulation"]: + result["insulation_thickness"] = "above average" + elif result["internal_insulation"]: + result["insulation_thickness"] = "above average" + else: + result["insulation_thickness"] = "average" + return result diff --git a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py index 5738f77f..280e7459 100644 --- a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py @@ -1,14 +1,14 @@ clean_floor_cases = [ - {'original_description': '(another dwelling below)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False, + {'original_description': '(another dwelling below)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, - {'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_assumed': False, 'is_to_unheated_space': False, + {'original_description': '(anheddiad arall islaw)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, - {'original_description': '(other premises below)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(other premises below)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, @@ -342,8 +342,8 @@ clean_floor_cases = [ {'original_description': 'To unheated space, no insulation (assumed)', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False}, - {'original_description': '(eiddo arall islaw)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(eiddo arall islaw)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': None, "another_property_below": True}, diff --git a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py index ee7f865b..6b719afd 100644 --- a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py @@ -1,10 +1,11 @@ clean_roof_test_cases = [ - {'original_description': '(another dwelling above)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, + {'original_description': '(another dwelling above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, + 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, - {'original_description': '(other premises above)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, + {'original_description': '(other premises above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, @@ -362,8 +363,9 @@ clean_roof_test_cases = [ 'thermal_transmittance_unit': None, 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'}, - {'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': None, - 'thermal_transmittance_unit': None, 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, + {'original_description': '(eiddo arall uwchben)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': "w/m-¦k", 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, + 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, 'insulation_thickness': None}, {'original_description': 'Ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)', 'thermal_transmittance': None, diff --git a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py index 40d6fb9c..300702a7 100644 --- a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py @@ -567,17 +567,17 @@ wall_cases = [ {'original_description': 'Cavity wall, filled cavity', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None, + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average", 'external_insulation': False, 'internal_insulation': False}, {'original_description': 'Cavity wall, filled cavity and external insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': True, 'internal_insulation': False}, {'original_description': 'Cavity wall, filled cavity and internal insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': False, 'internal_insulation': True}, {'original_description': 'Cavity wall, with external insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': False, 'is_solid_brick': False, @@ -723,7 +723,7 @@ wall_cases = [ {'original_description': 'Waliau ceudod, ceudod wediGÇÖi lenwi', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None, + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': "average", 'external_insulation': False, 'internal_insulation': False}, {'original_description': 'Waliau ceudod, fel yGÇÖu hadeiladwyd, wediGÇÖu hinswleiddio (rhagdybiaeth)', 'thermal_transmittance': None, @@ -778,7 +778,7 @@ wall_cases = [ {'original_description': 'Waliau ceudod, ynysydd allanol a llenwi ceudod', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': True, 'is_filled_cavity': True, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, - 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'average', + 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'above average', 'external_insulation': True, 'internal_insulation': False}, {'original_description': 'Gwenithfaen neu risgraig, gydag inswleiddio mewnol', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, diff --git a/etl/epc_clean/tests/test_roof_attributes.py b/etl/epc_clean/tests/test_roof_attributes.py index 54b59f1a..b0663a3e 100644 --- a/etl/epc_clean/tests/test_roof_attributes.py +++ b/etl/epc_clean/tests/test_roof_attributes.py @@ -75,8 +75,8 @@ class TestRoofAttributes: "is_assumed": False, "is_flat": False, "is_thatched": False, - "thermal_transmittance": None, - "thermal_transmittance_unit": None, + "thermal_transmittance": 0, + "thermal_transmittance_unit": "w/m-¦k", } for k in expected_output: From d5b2ff9c3617b8aea176e7dc672d8ac1498bf95a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Oct 2023 22:26:51 +1100 Subject: [PATCH 08/75] built new dataset for testing --- etl/epc/property_change_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 300c0417..6e724e33 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -622,7 +622,7 @@ def app(): save_dataframe_to_s3_parquet( df=output, bucket_name="retrofit-data-dev", - file_key="sap_change_model/dataset.parquet", + file_key="sap_change_model/dataset_test.parquet", ) From f0db6b69df37159c52abd16611472d02b4b6e31d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 10:14:52 +1100 Subject: [PATCH 09/75] created a small dataset of uprn and directory name to keep track of all equal rows --- etl/epc/property_change_app.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 6e724e33..a6034e3d 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -411,8 +411,8 @@ def app(): dataset = [] cleaning_dataset = [] - # Keep track of the number of all equals - all_equal_count = 0 + # Keep track of the all equals + all_equal_rows = [] for directory in tqdm(directories): @@ -521,8 +521,8 @@ def app(): ) if all_equal: - # Keep track of this for the moment - all_equal_count += 1 + # Keep track of this for the moment so we can analyse + all_equal_rows.append({"uprn": uprn, "directory_name": directory.name}) continue features = pd.concat([starting_record, ending_record]) @@ -622,7 +622,7 @@ def app(): save_dataframe_to_s3_parquet( df=output, bucket_name="retrofit-data-dev", - file_key="sap_change_model/dataset_test.parquet", + file_key="sap_change_model/dataset.parquet", ) From 0e7f56e35632401e7e54a96034f604160a3cbc67 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 10:28:18 +1100 Subject: [PATCH 10/75] Added all_equal_rows storage --- etl/epc/property_change_app.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index a6034e3d..435b668d 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -597,7 +597,7 @@ def app(): cleaning_averages["LOCAL_AUTHORITY"] = df["LOCAL_AUTHORITY"].values[0] cleaning_dataset.append(cleaning_averages) - print("Final all equal count: %s" % str(all_equal_count)) + print("Final all equal count: %s" % str(len(all_equal_rows))) # Store cleaning dataset in s3 as a parquet file cleaning_dataset = pd.concat(cleaning_dataset) @@ -625,6 +625,14 @@ def app(): file_key="sap_change_model/dataset.parquet", ) + # Store all_equal_rows + all_equal_rows = pd.DataFrame(all_equal_rows) + save_dataframe_to_s3_parquet( + df=all_equal_rows, + bucket_name="retrofit-data-dev", + file_key="sap_change_model/all_equal_rows.parquet", + ) + if __name__ == "__main__": app() From 8b078da71ec1c614955730630d873a3e54d2d01c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 10:58:45 +1100 Subject: [PATCH 11/75] Allow system built walls to have iwi and ewi --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 6 ++ recommendations/WallRecommendations.py | 63 +------------------ .../tests/test_wall_recommendations.py | 25 ++++++++ 5 files changed, 35 insertions(+), 63 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index ca0e1cd9..3b05c6ac 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index ab0e32b5..400b2dee 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -122,6 +122,12 @@ async def trigger_plan(body: PlanTriggerRequest): # import pickle # with open("input_properties.pickle", "rb") as f: # input_properties = pickle.load(f) + # + # with open("cleaned.pickle", "rb") as f: + # cleaned = pickle.load(f) + # + # with open("materials_by_type.pickle", "wb") as f: + # materials_by_type = pickle.load(f) recommendations = {} recommendations_scoring_data = [] diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 379f7026..66ecdf3b 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -112,7 +112,7 @@ class WallRecommendations(Definitions): self.estimated_u_value = u_value - if self.property.walls["is_solid_brick"]: + if self.property.walls["is_solid_brick"] | self.property.walls["is_system_built"]: if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: self.find_insulation(u_value) @@ -128,9 +128,7 @@ class WallRecommendations(Definitions): return - logger.error("Not implemented yet") - return - # NotImplementedError("Not implemented yet") + raise NotImplementedError("Not implemented yet") def find_cavity_insulation(self, u_value, insulation_thickness): """ @@ -273,63 +271,6 @@ class WallRecommendations(Definitions): self.recommendations += ewi_recommendations + iwi_recommendations - # We also can recommend both internal and external wall insulation together - # By looping through ewi first, if there is nothing there, that ensures not combinations are tested - for ewi_part in ewi_parts: - for iwi_part in iwi_parts: - for (ewi_depth, ewi_cost_per_unit), (iwi_depth, iwi_cost_per_unit) in itertools.product( - zip(ewi_part["depths"], ewi_part["cost"]), - zip(iwi_part["depths"], iwi_part["cost"]) - ): - ewi_part_u_value = r_value_per_mm_to_u_value(ewi_depth, ewi_part["r_value_per_mm"]) - iwi_part_u_value = r_value_per_mm_to_u_value(iwi_depth, iwi_part["r_value_per_mm"]) - - # First calculate the new U-value after applying external wall insulation - _, ewi_new_u_value = calculate_u_value_uplift(u_value, ewi_part_u_value) - # Then calculate the new U-value after applying internal wall insulation - _, combined_new_u_value = calculate_u_value_uplift(ewi_new_u_value, iwi_part_u_value) - combined_new_u_value = round(combined_new_u_value, 2) - - if combined_new_u_value < self.DIMINISHING_RETURNS_U_VALUE: - # We don't recommend an overkill solution - continue - - # Check if the combined new U-value meets the requirement - if combined_new_u_value - self.U_VALUE_ERROR <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - # Here you might want to define a way to add both recommendations together. - # For now, I'm adding them as separate items in the list - ewi_esimtated_cost = ewi_cost_per_unit * self.property.insulation_wall_area - iwi_esimtated_cost = iwi_cost_per_unit * self.property.insulation_wall_area - - recommendation = { - "parts": [ - get_recommended_part( - part=ewi_part, - selected_depth=ewi_depth, - quantity=self.property.insulation_wall_area, - quantity_unit=QuantityUnits.m2.value, - selected_total_cost=ewi_esimtated_cost - ), - get_recommended_part( - part=iwi_part, - selected_depth=iwi_depth, - quantity=self.property.insulation_wall_area, - quantity_unit=QuantityUnits.m2.value, - selected_total_cost=iwi_esimtated_cost - ) - ], - "type": "wall_insulation", - "description": ( - "Install " + self._make_description(ewi_part, ewi_depth) + " and " + - self._make_description(iwi_part, iwi_depth) - ), - "starting_u_value": u_value, - "new_u_value": combined_new_u_value, - "sap_points": None, - "cost": ewi_esimtated_cost + iwi_esimtated_cost, - } - self.recommendations.append(recommendation) - self.prune_diminishing_recommendations() @staticmethod diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index e910a8f5..c25c00f9 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -503,3 +503,28 @@ class TestCavityWallRecommensations: assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.57) assert np.isclose(recommender.recommendations[1]["cost"], 1250) + + def test_system_built_wall(self): + input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property2.walls = { + 'original_description': 'System built, as built, no insulation (assumed)', + 'clean_description': 'System built, as built, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, + 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, + 'is_system_built': True, 'is_timber_frame': False, 'is_granite_or_whinstone': False, + 'is_as_built': True, 'is_cob': False, 'is_assumed': True, + 'is_sandstone_or_limestone': False, 'is_park_home': False, + 'insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False + } + input_property2.age_band = "F" + input_property2.insulation_wall_area = 120 + + recommender2 = WallRecommendations( + property_instance=input_property2, + materials=internal_wall_insulation_parts + external_wall_insulation_parts + ) + + assert not recommender2.recommendations + + recommender2.recommend() \ No newline at end of file From 8261a61454bc8de9f828f6afbb060e74ae102554 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 14:59:02 +1100 Subject: [PATCH 12/75] added new wall recommendations tests --- recommendations/WallRecommendations.py | 17 ++- .../tests/test_wall_recommendations.py | 108 +++++++++++++++++- 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 66ecdf3b..b3eea7e0 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -112,15 +112,6 @@ class WallRecommendations(Definitions): self.estimated_u_value = u_value - if self.property.walls["is_solid_brick"] | self.property.walls["is_system_built"]: - - if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - self.find_insulation(u_value) - return - - # If the u-value is within regulations, we don't do anything - return - if is_cavity_wall: if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: # Test filling cavity @@ -128,7 +119,13 @@ class WallRecommendations(Definitions): return - raise NotImplementedError("Not implemented yet") + # Remaining wall types are treated with IWI or EWI + if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + self.find_insulation(u_value) + return + + # If the u-value is within regulations, we don't do anything + return def find_cavity_insulation(self, u_value, insulation_thickness): """ diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index c25c00f9..334c02b7 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -519,6 +519,10 @@ class TestCavityWallRecommensations: } input_property2.age_band = "F" input_property2.insulation_wall_area = 120 + input_property2.restricted_measures = False + input_property2.data = {"property-type": "house"} + + assert input_property2.walls["is_system_built"] recommender2 = WallRecommendations( property_instance=input_property2, @@ -527,4 +531,106 @@ class TestCavityWallRecommensations: assert not recommender2.recommendations - recommender2.recommend() \ No newline at end of file + recommender2.recommend() + + assert recommender2.recommendations + assert len(recommender2.recommendations) == 6 + assert recommender2.estimated_u_value == 1 + assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.29) + assert np.isclose(recommender2.recommendations[0]["cost"], 10800) + assert recommender2.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" + assert recommender2.recommendations[0]["parts"][0]["depths"] == [90] + + assert np.isclose(recommender2.recommendations[5]["new_u_value"], 0.29) + assert np.isclose(recommender2.recommendations[5]["cost"], 2400) + assert recommender2.recommendations[5]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender2.recommendations[5]["parts"][0]["depths"] == [20] + + assert np.isclose(recommender2.recommendations[3]["new_u_value"], 0.28) + assert np.isclose(recommender2.recommendations[3]["cost"], 4800) + assert recommender2.recommendations[3]["parts"][0]["type"] == "external_wall_insulation" + assert recommender2.recommendations[3]["parts"][0]["depths"] == [40] + + def test_timber_frame_wall(self): + input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property3.walls = { + 'original_description': 'Timber frame, as built, no insulation (assumed)', + 'clean_description': 'Timber frame, as built, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, + 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, + 'is_system_built': False, 'is_timber_frame': True, 'is_granite_or_whinstone': False, + 'is_as_built': True, 'is_cob': False, 'is_assumed': True, + 'is_sandstone_or_limestone': False, 'is_park_home': False, + 'insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False + } + input_property3.age_band = "B" + input_property3.insulation_wall_area = 99 + input_property3.restricted_measures = False + input_property3.data = {"property-type": "house"} + + assert input_property3.walls["is_timber_frame"] + + recommender3 = WallRecommendations( + property_instance=input_property3, + materials=internal_wall_insulation_parts + external_wall_insulation_parts + ) + + assert not recommender3.recommendations + + recommender3.recommend() + + assert recommender3.recommendations + assert len(recommender3.recommendations) == 2 + assert recommender3.estimated_u_value == 1.9 + assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.26) + assert np.isclose(recommender3.recommendations[0]["cost"], 12375) + assert recommender3.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" + assert recommender3.recommendations[0]["parts"][0]["depths"] == [125] + + assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.26) + assert np.isclose(recommender3.recommendations[1]["cost"], 4950) + assert recommender3.recommendations[1]["parts"][0]["type"] == "external_wall_insulation" + assert recommender3.recommendations[1]["parts"][0]["depths"] == [50] + + def test_granite_or_whinstone_wall(self): + input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property4.walls = { + 'original_description': 'Granite or whinstone, as built, no insulation (assumed)', + 'clean_description': 'Granite or whinstone, as built, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, + 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, + 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': True, + 'is_as_built': True, 'is_cob': False, 'is_assumed': True, + 'is_sandstone_or_limestone': False, 'is_park_home': False, + 'insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False + } + input_property4.age_band = "A" + input_property4.insulation_wall_area = 223 + input_property4.restricted_measures = False + input_property4.data = {"property-type": "Bungalow"} + + assert input_property4.walls["is_granite_or_whinstone"] + + recommender4 = WallRecommendations( + property_instance=input_property4, + materials=internal_wall_insulation_parts + external_wall_insulation_parts + ) + + assert not recommender4.recommendations + + recommender4.recommend() + + assert recommender4.recommendations + assert len(recommender4.recommendations) == 2 + assert recommender4.estimated_u_value == 2.3 + assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.27) + assert np.isclose(recommender4.recommendations[0]["cost"], 27875) + assert recommender4.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" + assert recommender4.recommendations[0]["parts"][0]["depths"] == [125] + + assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.27) + assert np.isclose(recommender4.recommendations[1]["cost"], 11150) + assert recommender4.recommendations[1]["parts"][0]["type"] == "external_wall_insulation" + assert recommender4.recommendations[1]["parts"][0]["depths"] == [50] From 061ee27ffdf28412f914ac80db501ed7b74d4c4a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 15:03:52 +1100 Subject: [PATCH 13/75] Added cob wall unit test --- .../tests/test_wall_recommendations.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index 334c02b7..81d040c5 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -634,3 +634,52 @@ class TestCavityWallRecommensations: assert np.isclose(recommender4.recommendations[1]["cost"], 11150) assert recommender4.recommendations[1]["parts"][0]["type"] == "external_wall_insulation" assert recommender4.recommendations[1]["parts"][0]["depths"] == [50] + + def test_cob_wall(self): + input_property5 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property5.walls = { + 'original_description': 'Cob, as built', + 'clean_description': 'Cob, as built', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, + 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, + 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, + 'is_as_built': False, 'is_cob': True, 'is_assumed': False, + 'is_sandstone_or_limestone': False, 'is_park_home': False, + 'insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False + } + input_property5.age_band = "E" + input_property5.insulation_wall_area = 77 + input_property5.restricted_measures = False + input_property5.data = {"property-type": "Bungalow"} + + assert input_property5.walls["is_cob"] + + recommender5 = WallRecommendations( + property_instance=input_property5, + materials=internal_wall_insulation_parts + external_wall_insulation_parts + ) + + assert not recommender5.recommendations + + recommender5.recommend() + + assert recommender5.recommendations + assert len(recommender5.recommendations) == 9 + assert recommender5.estimated_u_value == 0.8 + assert np.isclose(recommender5.recommendations[0]["new_u_value"], 0.29) + assert np.isclose(recommender5.recommendations[0]["cost"], 6160) + assert recommender5.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" + assert recommender5.recommendations[0]["parts"][0]["depths"] == [80] + + assert np.isclose(recommender5.recommendations[3]["new_u_value"], 0.26) + assert np.isclose(recommender5.recommendations[3]["cost"], 7700) + assert recommender5.recommendations[3]["parts"][0]["type"] == "external_wall_insulation" + assert recommender5.recommendations[3]["parts"][0]["depths"] == [100] + + assert np.isclose(recommender5.recommendations[6]["new_u_value"], 0.26) + assert np.isclose(recommender5.recommendations[6]["cost"], 7700) + assert recommender5.recommendations[6]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender5.recommendations[6]["parts"][0]["depths"] == [100] + + From 08487d8f19e385c8d57907877c25f283bc2c7831 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 15:14:37 +1100 Subject: [PATCH 14/75] added sandstone or limestone unit tests --- .../tests/test_wall_recommendations.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index 81d040c5..d4ce5884 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -682,4 +682,49 @@ class TestCavityWallRecommensations: assert recommender5.recommendations[6]["parts"][0]["type"] == "internal_wall_insulation" assert recommender5.recommendations[6]["parts"][0]["depths"] == [100] + def test_sandstone_or_limestone_wall(self): + input_property6 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property6.walls = { + 'original_description': 'Sandstone or limestone, as built, no insulation (assumed)', + 'clean_description': 'Sandstone or limestone, as built, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, + 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, + 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, + 'is_as_built': False, 'is_cob': False, 'is_assumed': False, + 'is_sandstone_or_limestone': True, 'is_park_home': False, + 'insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False + } + input_property6.age_band = "F" + input_property6.insulation_wall_area = 350 + input_property6.restricted_measures = False + input_property6.data = {"property-type": "House"} + assert input_property6.walls["is_sandstone_or_limestone"] + + recommender6 = WallRecommendations( + property_instance=input_property6, + materials=internal_wall_insulation_parts + external_wall_insulation_parts + ) + + assert not recommender6.recommendations + + recommender6.recommend() + + assert recommender6.recommendations + assert len(recommender6.recommendations) == 6 + assert recommender6.estimated_u_value == 1 + assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.29) + assert np.isclose(recommender6.recommendations[0]["cost"], 31500) + assert recommender6.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" + assert recommender6.recommendations[0]["parts"][0]["depths"] == [90] + + assert np.isclose(recommender6.recommendations[2]["new_u_value"], 0.28) + assert np.isclose(recommender6.recommendations[2]["cost"], 35000) + assert recommender6.recommendations[2]["parts"][0]["type"] == "external_wall_insulation" + assert recommender6.recommendations[2]["parts"][0]["depths"] == [100] + + assert np.isclose(recommender6.recommendations[4]["new_u_value"], 0.28) + assert np.isclose(recommender6.recommendations[4]["cost"], 35000) + assert recommender6.recommendations[4]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender6.recommendations[4]["parts"][0]["depths"] == [100] From 8c5429c5d666f27c0d4cef1199d9e09d80f8866b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 15:52:55 +1100 Subject: [PATCH 15/75] Fixed broken tests --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/utils.py | 16 ++++----- backend/tests/test_sap_model_prep.py | 27 +++++++------- .../tests/test_data/wall_uvalue_test_cases.py | 8 ++--- .../tests/test_floor_recommendations.py | 2 ++ .../tests/test_wall_recommendations.py | 36 +++---------------- 7 files changed, 32 insertions(+), 61 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 3b05c6ac..ca0e1cd9 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 0565ab94..85a02b61 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -86,7 +86,7 @@ def create_recommendation_scoring_data( } # Set staring u-values if we don't have them - if not scoring_dict["walls_thermal_transmittance"]: + if scoring_dict["walls_thermal_transmittance"] is None: scoring_dict["walls_thermal_transmittance"] = get_wall_u_value( clean_description=property.walls["clean_description"], age_band=property.age_band, @@ -94,7 +94,7 @@ def create_recommendation_scoring_data( is_sandstone_or_limestone=property.walls["is_sandstone_or_limestone"] ) - if not scoring_dict["floor_thermal_transmittance"]: + if scoring_dict["floor_thermal_transmittance"] is None: scoring_dict["floor_thermal_transmittance"] = get_floor_u_value( floor_type=property.floor_type, area=property.floor_area, @@ -104,7 +104,7 @@ def create_recommendation_scoring_data( age_band=property.age_band, ) - if not scoring_dict["roof_thermal_transmittance"]: + if scoring_dict["roof_thermal_transmittance"] is None: scoring_dict["roof_thermal_transmittance"] = get_roof_u_value( insulation_thickness=property.roof["insulation_thickness"], has_dwelling_above=property.roof["has_dwelling_above"], @@ -130,7 +130,7 @@ def create_recommendation_scoring_data( scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"] scoring_dict["walls_insulation_thickness_ENDING"] = "above average" else: - if not scoring_dict["walls_thermal_transmittance_ENDING"]: + if scoring_dict["walls_thermal_transmittance_ENDING"] is None: scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value( clean_description=property.walls["clean_description"], age_band=property.age_band, @@ -151,7 +151,7 @@ def create_recommendation_scoring_data( # We don't really see above average for this in the training data scoring_dict["floor_insulation_thickness_ENDING"] = "average" else: - if not scoring_dict["floor_thermal_transmittance_ENDING"]: + if scoring_dict["floor_thermal_transmittance_ENDING"] is None: scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value( floor_type=property.floor_type, area=property.floor_area, @@ -167,7 +167,7 @@ def create_recommendation_scoring_data( if recommendation["type"] not in ["wall_insulation", "floor_insulation"]: raise NotImplementedError("Implement me") - if not scoring_dict["roof_thermal_transmittance_ENDING"]: + if scoring_dict["roof_thermal_transmittance_ENDING"] is None: scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value( insulation_thickness=property.roof["insulation_thickness"], has_dwelling_above=property.roof["has_dwelling_above"], @@ -180,7 +180,7 @@ def create_recommendation_scoring_data( is_at_rafters=property.roof["is_at_rafters"], ) - if scoring_dict["roof_insulation_thickness_ENDING"] is None: - scoring_dict["roof_insulation_thickness_ENDING"] = "none" + if scoring_dict["roof_insulation_thickness_ENDING"] is None: + scoring_dict["roof_insulation_thickness_ENDING"] = "none" return scoring_dict diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 4f36dbe8..c11a0fae 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -1,15 +1,16 @@ from backend.Property import Property from etl.epc.DataProcessor import DataProcessor -from backend.app.plan.utils import create_recommendation_scoring_data +from backend.app.plan.utils import create_recommendation_scoring_data, get_cleaned from etl.epc.settings import COLUMNS_TO_MERGE_ON from epc_api.client import EpcClient import pandas as pd -import os import pytest -import pickle + +from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm +# import pickle # with open("sap_change_dataset.pickle", "rb") as f: # sap_change_dataset = pickle.load(f) # @@ -109,18 +110,14 @@ from tqdm import tqdm class TestSapModelPrep: @pytest.fixture - def cleaned(self): - with open( - os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaned.pickle", "rb" - ) as f: - return pickle.load(f) + def cleaning_data(self): + return read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", + ) @pytest.fixture - def cleaning_data(self): - with open( - os.path.abspath(os.path.dirname(__file__)) + "/test_data/cleaning_data.pickle", "rb" - ) as f: - return pickle.load(f) + def cleaned(self): + return get_cleaned() def test_fill_cavity_wall(self, cleaned, cleaning_data): """ @@ -395,7 +392,7 @@ class TestSapModelPrep: 'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0, 'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307, - 'walls_thermal_transmittance': 0.45, 'is_cavity_wall': False, 'is_filled_cavity': False, + 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', @@ -482,7 +479,7 @@ class TestSapModelPrep: ending_lodgement_date2 = '2020-11-24' - starting_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) + ending_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) recommendation2 = { "recommendation_id": 0, diff --git a/recommendations/tests/test_data/wall_uvalue_test_cases.py b/recommendations/tests/test_data/wall_uvalue_test_cases.py index 1cc6823c..e0c6ebe3 100644 --- a/recommendations/tests/test_data/wall_uvalue_test_cases.py +++ b/recommendations/tests/test_data/wall_uvalue_test_cases.py @@ -4,21 +4,21 @@ wall_uvalue_test_cases = [ "age_band": "A", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.7 + "uvalue": 1.3 }, { "clean_description": "Cavity wall, as built, partial insulation", "age_band": "F", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.4 + "uvalue": 0.85 }, { "clean_description": "Cavity wall, as built, partial insulation", - "age_band": "F", + "age_band": "G", "is_granite_or_whinstone": False, "is_sandstone_or_limestone": False, - "uvalue": 0.4 + "uvalue": 0.5375 }, { diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index f3df168f..f34bbe81 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -118,6 +118,7 @@ class TestFloorRecommendations: input_properties[2].age_band = "A" input_properties[2].perimeter = 20 input_properties[2].wall_type = "solid brick" + input_properties[2].floor_type = "suspended" recommender = FloorRecommendations( property_instance=input_properties[2], @@ -160,6 +161,7 @@ class TestFloorRecommendations: input_properties[4].age_band = "B" input_properties[4].perimeter = 50 input_properties[4].wall_type = "solid brick" + input_properties[4].floor_type = "solid" recommender = FloorRecommendations( property_instance=input_properties[4], diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index d4ce5884..3663364c 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -407,36 +407,8 @@ class TestWallRecommendationsBase: wall_recommendations_instance.property.data = {"property-type": "house"} assert wall_recommendations_instance.ewi_valid is True - def test_recommend_without_u_value(self, wall_recommendations_instance): - wall_recommendations_instance.property.walls = { - "thermal_transmittance": None, - "is_solid_brick": False, - "is_cavity_wall": False, - "insulation_thickness": "none", - "clean_description": "Solid brick, as built, no insulation", - "is_granite_or_whinstone": False, - "is_sandstone_or_limestone": False, - } - wall_recommendations_instance.property.age_band = "A" - with pytest.raises(NotImplementedError): - wall_recommendations_instance.recommend() - class TestCavityWallRecommensations: - data = { - 'low-energy-fixed-light-count': '', 'address': '123 Fake Street', - 'floor-height': '', 'construction-age-band': 'England and Wales: 1950-1966', - 'address3': '', 'property-type': 'House', 'local-authority-label': 'Melton', - 'county': 'Leicestershire', 'postcode': 'LE14 2QH', - 'solar-water-heating-flag': 'N', 'constituency': 'E14000909', - 'number-heated-rooms': '5', 'local-authority': 'E07000133', 'built-form': 'End-Terrace', - 'address1': '1, 23 fake', 'total-floor-area': '85.0', 'environment-impact-current': '49', - 'number-habitable-rooms': 3, 'address2': 'Fake', 'posttown': 'IDK', - 'walls-energy-eff': 'Poor', 'current-energy-rating': 'D', - 'transaction-type': 'ECO assessment', 'uprn': '999', 'current-energy-efficiency': '57', - 'lodgement-date': '2019-07-10', 'lmk-key': '999', 'tenure': 'rental (private)', 'floor-level': 'NODATA!', - 'walls-description': 'Cavity wall, as built, no insulation (assumed)', - } def test_fill_empty_cavity(self): input_property = Property(id=1, postcode="F4k3", address1="123 fake street", epc_client=Mock()) @@ -465,10 +437,10 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.5 - assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.25) + assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.37) assert np.isclose(recommender.recommendations[0]["cost"], 1000) - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.26) + assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.38) assert np.isclose(recommender.recommendations[1]["cost"], 1250) def test_fill_partial_filled_cavity(self): @@ -498,10 +470,10 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.3 - assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.56) + assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.43) assert np.isclose(recommender.recommendations[0]["cost"], 1000) - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.57) + assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.45) assert np.isclose(recommender.recommendations[1]["cost"], 1250) def test_system_built_wall(self): From ada2101a735e41844504b114fa216a3349637081 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 17:25:50 +1100 Subject: [PATCH 16/75] Wrote ventilation recommendations, still needs unit tests --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 2 + backend/app/plan/router.py | 14 +++- backend/app/plan/utils.py | 9 ++- recommendations/VentilationRecommendations.py | 70 +++++++++++++++++++ recommendations/WallRecommendations.py | 1 - 7 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 recommendations/VentilationRecommendations.py diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index ca0e1cd9..3b05c6ac 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 09d7369d..cf6dd971 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -13,6 +13,7 @@ class MaterialType(enum.Enum): external_wall_insulation = "external_wall_insulation" internal_wall_insulation = "internal_wall_insulation" cavity_wall_insulation = "cavity_wall_insulation" + mechanical_ventilation = "mechanical_ventilation" class DepthUnit(enum.Enum): @@ -21,6 +22,7 @@ class DepthUnit(enum.Enum): class CostUnit(enum.Enum): gbp_sq_meter = "gbp_sq_meter" + gbp_per_unit = "gbp_per_unit" class RValueUnit(enum.Enum): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 400b2dee..135b877d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,6 +30,7 @@ from backend.Property import Property from etl.epc.DataProcessor import DataProcessor from etl.epc.settings import COLUMNS_TO_MERGE_ON from recommendations.FloorRecommendations import FloorRecommendations +from recommendations.VentilationRecommendations import VentilationRecommendations from recommendations.optimiser.CostOptimiser import CostOptimiser from recommendations.optimiser.GainOptimiser import GainOptimiser from recommendations.optimiser.optimiser_functions import prepare_input_measures @@ -125,9 +126,6 @@ async def trigger_plan(body: PlanTriggerRequest): # # with open("cleaned.pickle", "rb") as f: # cleaned = pickle.load(f) - # - # with open("materials_by_type.pickle", "wb") as f: - # materials_by_type = pickle.load(f) recommendations = {} recommendations_scoring_data = [] @@ -160,6 +158,16 @@ async def trigger_plan(body: PlanTriggerRequest): if wall_recomender.recommendations: property_recommendations.append(wall_recomender.recommendations) + # Ventilation recommendations + ventilation_recomender = VentilationRecommendations( + property_instance=p, + materials=materials_by_type["ventilation"] + ) + ventilation_recomender.recommend() + + if ventilation_recomender.recommendation: + property_recommendations.append(ventilation_recomender.recommendation) + # We insert temporary ids into the recommendations which is important for the optimiser later property_recommendations = insert_temp_recommendation_id(property_recommendations) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 85a02b61..0a1eaa72 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -15,7 +15,8 @@ def filter_materials(materials): mapping = { "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], - "floor": ["suspended_floor_insulation", "solid_floor_insulation"] + "floor": ["suspended_floor_insulation", "solid_floor_insulation"], + "ventilation": ["mechanical_ventilation"], } materials = [row2dict(material) for material in materials] @@ -164,9 +165,13 @@ def create_recommendation_scoring_data( if scoring_dict["floor_insulation_thickness_ENDING"] is None: scoring_dict["floor_insulation_thickness_ENDING"] = "none" - if recommendation["type"] not in ["wall_insulation", "floor_insulation"]: + if recommendation["type"] == "mechanical_ventilation": + scoring_dict["MECHANICAL_VENTILATION_ENDING"] = 'mechanical, extract only' + + if recommendation["type"] not in ["wall_insulation", "floor_insulation", "mechanical_ventilation"]: raise NotImplementedError("Implement me") + # Fill missing roof u-values - this fill is not based on recommended upgrades if scoring_dict["roof_thermal_transmittance_ENDING"] is None: scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value( insulation_thickness=property.roof["insulation_thickness"], diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py new file mode 100644 index 00000000..35de9b3b --- /dev/null +++ b/recommendations/VentilationRecommendations.py @@ -0,0 +1,70 @@ +import pandas as pd +from BaseUtility import Definitions +from backend.Property import Property + + +class VentilationRecommendations(Definitions): + """ + For properties that do not have ventilation, we recommend installing ventilaion + This is particularly important for properties that have insulated walls and is also + crucial for prevent overheating risks in warmer months + """ + + VENTILATION_DESCRIPTIONS = [ + 'mechanical, extract only', + 'mechanical, supply and extract' + ] + + def __init__( + self, + property_instance: Property, + materials + ): + self.property = property_instance + + self.has_ventilaion = None + self.recommendation = None + self.materials = materials + + def identify_ventilation(self): + self.has_ventilaion = self.property.data["mechanical-ventilation"] in self.VENTILATION_DESCRIPTIONS + + def recommend(self): + """ + If there is no ventilation, we recommend installing ventilation + + Generally, best practice is to install controlled ventilation for insulated walls so we still recommend + ventilation if there is natural ventilation + :return: + """ + + self.identify_ventilation() + if self.has_ventilaion: + return + + if len(self.materials) != 1: + raise NotImplementedError("Only handled the case of having one venilation option") + + # We recommend installing 2 units + n_units = 2 + + part = self.materials.copy() + + estimated_cost = n_units * part[0]["cost"] + + part[0]["estimated_cost"] = estimated_cost + part[0]["quantity"] = n_units + part[0]["quantity_unit"] = None + + # We recommend installing two mechanical ventilation systems + self.recommendation = [ + { + "parts": part, + "type": part[0]["type"], + "description": "Install %s" % part[0]["description"], + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + "cost": estimated_cost, + } + ] diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index b3eea7e0..ad2ca861 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -1,4 +1,3 @@ -import itertools import math from typing import List From 65c362e0ac810d7e4c6ba863387ff8c4fa0d655b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 17:38:54 +1100 Subject: [PATCH 17/75] Added ventilation unit testa --- .../tests/test_ventilation_recommendations.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 recommendations/tests/test_ventilation_recommendations.py diff --git a/recommendations/tests/test_ventilation_recommendations.py b/recommendations/tests/test_ventilation_recommendations.py new file mode 100644 index 00000000..2dcaba57 --- /dev/null +++ b/recommendations/tests/test_ventilation_recommendations.py @@ -0,0 +1,110 @@ +from backend.Property import Property +from unittest.mock import Mock +from recommendations.VentilationRecommendations import VentilationRecommendations + +ventilation_materials = [ + { + 'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', + 'depths': None, 'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, + 'r_value_unit': None, 'thermal_conductivity': None, 'thermal_conductivity_unit': None, + 'link': None, 'is_active': True, 'estimated_cost': 1000, 'quantity': 2, 'quantity_unit': None + } +] + + +class TestVentilationRecommendations: + + def test_natural_ventilation(self): + input_property1 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property1.data = {"mechanical-ventilation": "natural"} + + recommender = VentilationRecommendations( + property_instance=input_property1, + materials=ventilation_materials + ) + + assert not recommender.recommendation + + recommender.recommend() + + assert len(recommender.recommendation) == 1 + + assert recommender.recommendation[0]["cost"] == 1000 + assert recommender.recommendation[0]["type"] == "mechanical_ventilation" + assert len(recommender.recommendation[0]["parts"]) == 1 + assert recommender.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' + assert recommender.recommendation[0]["parts"][0]["quantity"] == 2 + + def test_missing_ventilation(self): + input_property2 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property2.data = {"mechanical-ventilation": None} + + recommender2 = VentilationRecommendations( + property_instance=input_property2, + materials=ventilation_materials + ) + + assert not recommender2.recommendation + + recommender2.recommend() + + assert len(recommender2.recommendation) == 1 + + assert recommender2.recommendation[0]["cost"] == 1000 + assert recommender2.recommendation[0]["type"] == "mechanical_ventilation" + assert len(recommender2.recommendation[0]["parts"]) == 1 + assert recommender2.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' + assert recommender2.recommendation[0]["parts"][0]["quantity"] == 2 + + def test_nodata_ventilation(self): + input_property3 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property3.data = {"mechanical-ventilation": "NO DATA!!"} + + recommender3 = VentilationRecommendations( + property_instance=input_property3, + materials=ventilation_materials + ) + + assert not recommender3.recommendation + + recommender3.recommend() + + assert len(recommender3.recommendation) == 1 + + assert recommender3.recommendation[0]["cost"] == 1000 + assert recommender3.recommendation[0]["type"] == "mechanical_ventilation" + assert len(recommender3.recommendation[0]["parts"]) == 1 + assert recommender3.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' + assert recommender3.recommendation[0]["parts"][0]["quantity"] == 2 + + def test_existing_ventilation_1(self): + input_property4 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property4.data = {"mechanical-ventilation": 'mechanical, extract only'} + + recommender4 = VentilationRecommendations( + property_instance=input_property4, + materials=ventilation_materials + ) + + assert not recommender4.recommendation + + recommender4.recommend() + + assert not recommender4.recommendation + assert recommender4.has_ventilaion + + def test_existing_ventilation_2(self): + input_property5 = Property(id=1, postcode="F4k3 6", address1="623 fake street", epc_client=Mock()) + input_property5.data = {"mechanical-ventilation": 'mechanical, supply and extract'} + + recommender5 = VentilationRecommendations( + property_instance=input_property5, + materials=ventilation_materials + ) + + assert not recommender5.recommendation + + recommender5.recommend() + + assert not recommender5.recommendation + assert recommender5.has_ventilaion From e21142db8dec1bfdb176e7993876a7c9db66344e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 19 Oct 2023 08:43:21 +1100 Subject: [PATCH 18/75] added ventilation unit test to test_sap_model_prep --- backend/tests/test_sap_model_prep.py | 214 ++++++++++++++++++++++++--- 1 file changed, 195 insertions(+), 19 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index c11a0fae..1ba59fa0 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -10,18 +10,19 @@ from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm +# Handy code for selecting testin data # import pickle +# # with open("sap_change_dataset.pickle", "rb") as f: # sap_change_dataset = pickle.load(f) # # search_from = sap_change_dataset[ -# (sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]) & -# (sap_change_dataset["is_solid_brick"]) -# ] +# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) +# ] # search_from = search_from[ # (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & # (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & -# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & +# (search_from["MECHANICAL_VENTILATION_ENDING"] != search_from["MECHANICAL_VENTILATION_STARTING"]) & # (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & # (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) # ] @@ -51,16 +52,8 @@ from tqdm import tqdm # starting_cols.append(starting_col) # # # We want them to be different -# if c == "walls_thermal_transmittance_ENDING": -# if row[c] == row[starting_col]: -# same = False -# break -# else: -# continue -# -# # We want them to be different -# if c == "walls_insulation_thickness_ENDING": -# if row[c] == row[starting_col]: +# if c == "MECHANICAL_VENTILATION_ENDING": +# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): # same = False # break # else: @@ -78,20 +71,20 @@ from tqdm import tqdm # # import pandas as pd # -# start = row[starting_cols] +# start = row[["SAP_STARTING"] + starting_cols] # start.index = [c.replace("_STARTING", "") for c in start.index] -# end = row[ending_cols] +# end = row[["SAP_ENDING"] + ending_cols] # end.index = [c.replace("_ENDING", "") for c in end.index] # start["type"] = "starting" # end["type"] = "ending" # # compare = pd.concat([start, end], axis=1) # -# ending_lmk = "d0fc64d6b80db04c32998c9b846dd04c8f0b486231a11e4c062020b35af1312d" -# starting_lmk = "b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d" +# ending_lmk = "96b34dbe6cebdd7e648151e070047c8cd605c539851fc5a37e325903440081ab" +# starting_lmk = "dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247" # # client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY", "postcode": "E16 1BD"}) +# result = client.domestic.search(params={"address": "45 Shepperson Road", "postcode": "S6 4FG"}) # starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] # ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] @@ -106,6 +99,8 @@ from tqdm import tqdm # ) as f: # cleaning_data = pickle.load(f) +# TODO: Need to do floors, both suspended and solid + class TestSapModelPrep: @@ -530,3 +525,184 @@ class TestSapModelPrep: continue assert test_record2[c].values[0] == row2[c] + + def test_ventilation(self, cleaned, cleaning_data): + + starting_epc3 = { + 'low-energy-fixed-light-count': '', 'address': '45 Shepperson Road', 'uprn-source': 'Energy Assessor', + 'floor-height': '1.87', 'heating-cost-potential': '645', 'unheated-corridor-length': '', + 'hot-water-cost-potential': '69', 'construction-age-band': 'England and Wales: 1900-1929', + 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', + 'lighting-energy-eff': 'Average', 'environment-impact-potential': '75', + 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '1028', 'address3': '', + 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', + 'property-type': 'House', 'local-authority-label': 'Sheffield', 'fixed-lighting-outlets-count': '21', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96', + 'county': '', 'postcode': 'S6 4FG', 'solar-water-heating-flag': 'N', 'constituency': 'E14000921', + 'co2-emissions-potential': '2.9', 'number-heated-rooms': '5', + 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '152', + 'local-authority': 'E08000019', 'built-form': 'Enclosed Mid-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-06-13', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '59', 'address1': '45 Shepperson Road', + 'heat-loss-corridor': '', 'flat-storey-count': '', + 'constituency-label': 'Sheffield, Brightside and Hillsborough', 'roof-energy-eff': 'Very Poor', + 'total-floor-area': '107.0', 'building-reference-number': '10002892085', 'environment-impact-current': '46', + 'co2-emissions-current': '6.3', 'roof-description': 'Pitched, no insulation (assumed)', + 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', + 'posttown': 'SHEFFIELD', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 43% of fixed outlets', + 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '83', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', + 'main-heating-controls': '', 'lodgement-datetime': '2023-05-27 12:15:21', 'flat-top-storey': '', + 'current-energy-rating': 'E', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'marketed sale', 'uprn': '100051073214', 'current-energy-efficiency': '54', + 'energy-consumption-current': '335', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '131', 'lodgement-date': '2023-05-27', 'extension-count': '1', + 'mainheatc-env-eff': 'Average', + 'lmk-key': 'dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247', 'wind-turbine-count': '0', + 'tenure': 'Not defined - use in the case of a new dwelling for which the intended tenure in not known. It ' + 'is not to be used for an existing dwelling', + 'floor-level': '', 'potential-energy-efficiency': '80', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '43', + 'walls-description': 'Sandstone or limestone, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + } + + row3 = { + 'UPRN': '100051073214', 'RDSAP_CHANGE': 2, 'HEAT_DEMAND_CHANGE': -22, 'CARBON_CHANGE': -0.39999999999999947, + 'SAP_STARTING': 54, 'SAP_ENDING': 56, 'HEAT_DEMAND_STARTING': 335, 'HEAT_DEMAND_ENDING': 313, + 'CARBON_STARTING': 6.3, 'CARBON_ENDING': 5.9, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Mid-Terrace', + 'CONSTITUENCY': 'E14000921', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0, + 'FIXED_LIGHTING_OUTLETS_COUNT': 21.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1900-1929', + 'TRANSACTION_TYPE_STARTING': 'marketed sale', 'MECHANICAL_VENTILATION_STARTING': 'natural', + 'SECONDHEAT_DESCRIPTION_STARTING': 'None', 'ENERGY_TARIFF_STARTING': 'Single', + 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 0.0, + 'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, + 'LOW_ENERGY_LIGHTING_STARTING': 43.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, + 'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 107.0, 'FLOOR_HEIGHT_STARTING': 1.87, + 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'mechanical, extract only', + 'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'Single', + 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0, + 'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, + 'LOW_ENERGY_LIGHTING_ENDING': 43.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 1.0, + 'TOTAL_FLOOR_AREA_ENDING': 107.0, 'FLOOR_HEIGHT_ENDING': 1.87, 'DAYS_TO_STARTING': 3221, + 'DAYS_TO_ENDING': 2874, 'walls_thermal_transmittance': 2.0, 'is_cavity_wall': False, + 'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': True, + 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 2.0, 'is_park_home_ENDING': False, + 'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False, + 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.62, 'is_to_unheated_space': False, + 'is_to_external_air': False, 'is_suspended': True, 'is_solid': False, 'another_property_below': False, + 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.62, + 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 2.3, 'is_pitched': True, + 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'has_dwelling_above': False, 'roof_insulation_thickness': 'none', 'roof_thermal_transmittance_ENDING': 2.3, + 'roof_insulation_thickness_ENDING': 'none', 'heater_type': 'Unknown', 'system_type': 'from main system', + 'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', + 'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown', + 'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown', + 'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system', + 'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown', + 'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown', + 'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown', + 'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown', + 'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, + 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, + 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, + 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, + 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, + 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, + 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, + 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, + 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, + 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, + 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, + 'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False, + 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, + 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, + 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, + 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, + 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'Unknown', 'charging_system': 'Unknown', + 'switch_system': 'programmer', 'no_control': 'Unknown', 'dhw_control': 'Unknown', + 'community_heating': 'Unknown', 'multiple_room_thermostats': False, 'auxiliary_systems': 'bypass', + 'trvs': 'trvs', 'rate_control': 'Unknown', 'thermostatic_control_ENDING': 'Unknown', + 'charging_system_ENDING': 'Unknown', 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', + 'dhw_control_ENDING': 'Unknown', 'community_heating_ENDING': 'Unknown', + 'multiple_room_thermostats_ENDING': False, 'auxiliary_systems_ENDING': 'bypass', 'trvs_ENDING': 'trvs', + 'rate_control_ENDING': 'Unknown', 'glazing_type': 'double', 'glazing_type_ENDING': 'double', + 'fuel_type': 'mains gas', 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, + 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', + 'estimated_perimeter_STARTING': 41.634120622393354, 'estimated_perimeter_ENDING': 41.634120622393354 + } + + home3 = Property( + id=0, + postcode=starting_epc3["postcode"], + address1=starting_epc3["address1"], + epc_client=EpcClient(auth_token="notoken"), + data=starting_epc3 + ) + home3.get_components(cleaned) + + data_processor3 = DataProcessor(None, newdata=True) + data_processor3.insert_data(pd.DataFrame([home3.get_model_data()])) + + data_processor3.pre_process() + + starting_epc_data3 = data_processor3.get_component_features(suffix="_STARTING") + ending_epc_data3 = data_processor3.get_component_features(suffix="_ENDING") + fixed_data3 = data_processor3.get_fixed_features() + + ending_lodgement_date3 = '2022-06-14' + + ending_epc_data3["DAYS_TO_ENDING"] = data_processor3.calculate_days_to(ending_lodgement_date3) + + recommendation3 = { + "recommendation_id": 0, + "type": "mechanical_ventilation" + } + + test_record3 = create_recommendation_scoring_data( + property=home3, + recommendation=recommendation3, + starting_epc_data=starting_epc_data3, + ending_epc_data=ending_epc_data3, + fixed_data=fixed_data3, + ) + test_record3 = pd.DataFrame([test_record3]) + + # Test the final cleaning: + test_record3 = DataProcessor.apply_averages_cleaning( + data_to_clean=test_record3, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + ).drop(columns=["LOCAL_AUTHORITY"]) + + test_record3 = DataProcessor.clean_missings_after_description_process( + test_record3, [ + c for c in test_record3.columns if + ("thermal_transmittance" in c) or ("insulation_thickness" in c) + ] + ) + + for c in test_record3.columns: + if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: + continue + + assert test_record3[c].values[0] == row3[c] From 77a7035167887bbe51a1d1a40f1830bfdedf79f8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 19 Oct 2023 13:41:35 +1100 Subject: [PATCH 19/75] implemented number open fireplaces --- backend/app/plan/router.py | 10 +++++ backend/app/plan/utils.py | 7 ++- backend/tests/test_sap_model_prep.py | 24 ++++++++--- recommendations/FireplaceRecommendations.py | 48 +++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 recommendations/FireplaceRecommendations.py diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 135b877d..8c8f309b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -31,6 +31,7 @@ from etl.epc.DataProcessor import DataProcessor from etl.epc.settings import COLUMNS_TO_MERGE_ON from recommendations.FloorRecommendations import FloorRecommendations from recommendations.VentilationRecommendations import VentilationRecommendations +from recommendations.FireplaceRecommendations import FireplaceRecommendations from recommendations.optimiser.CostOptimiser import CostOptimiser from recommendations.optimiser.GainOptimiser import GainOptimiser from recommendations.optimiser.optimiser_functions import prepare_input_measures @@ -168,6 +169,15 @@ async def trigger_plan(body: PlanTriggerRequest): if ventilation_recomender.recommendation: property_recommendations.append(ventilation_recomender.recommendation) + # Fireplace sealing recommendations + fireplace_recommender = FireplaceRecommendations( + property_instance=p + ) + fireplace_recommender.recommend() + + if fireplace_recommender.recommendation: + property_recommendations.append(fireplace_recommender.recommendation) + # We insert temporary ids into the recommendations which is important for the optimiser later property_recommendations = insert_temp_recommendation_id(property_recommendations) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 0a1eaa72..2d9659e1 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -168,7 +168,12 @@ def create_recommendation_scoring_data( if recommendation["type"] == "mechanical_ventilation": scoring_dict["MECHANICAL_VENTILATION_ENDING"] = 'mechanical, extract only' - if recommendation["type"] not in ["wall_insulation", "floor_insulation", "mechanical_ventilation"]: + if recommendation["type"] == "sealing_open_fireplace": + scoring_dict["NUMBER_OPEN_FIREPLACES_ENDING"] = 0 + + if recommendation["type"] not in [ + "wall_insulation", "floor_insulation", "mechanical_ventilation", "sealing_open_fireplace" + ]: raise NotImplementedError("Implement me") # Fill missing roof u-values - this fill is not based on recommended upgrades diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 1ba59fa0..93567920 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -9,13 +9,25 @@ import pytest from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm - # Handy code for selecting testin data -# import pickle -# -# with open("sap_change_dataset.pickle", "rb") as f: -# sap_change_dataset = pickle.load(f) -# +import pickle + +with open("sap_change_dataset.pickle", "rb") as f: + sap_change_dataset = pickle.load(f) + +sap_change_dataset.columns.tolist() +z = sap_change_dataset[(sap_change_dataset["NUMBER_OPEN_FIREPLACES_ENDING"] > 0) & ( + sap_change_dataset["NUMBER_OPEN_FIREPLACES_STARTING"] > 0)].head(2).tail(1) +z["UPRN"] +z["SAP_STARTING"] +z["SAP_ENDING"] +z["NUMBER_OPEN_FIREPLACES_STARTING"] +z["NUMBER_OPEN_FIREPLACES_ENDING"] + + +# 10002083298 + +# m # search_from = sap_change_dataset[ # (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) # ] diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py new file mode 100644 index 00000000..3e82b9d1 --- /dev/null +++ b/recommendations/FireplaceRecommendations.py @@ -0,0 +1,48 @@ +import pandas as pd +from BaseUtility import Definitions +from backend.Property import Property + + +class FireplaceRecommendations(Definitions): + """ + For properties that have open fireplaces, we recommend sealing the fireplaces + """ + + # This is our base assumption for the cost of the work + COST_OF_WORK = 300 + + def __init__( + self, + property_instance: Property, + ): + self.property = property_instance + + self.has_ventilaion = None + self.recommendation = None + + def recommend(self): + """ + Based on the number of open fireplcaes found, we recommend sealing each one at a cost of + around £500 + :return: + """ + + number_open_fireplaces = int(self.property.data["number-open-fireplaces"]) + + if number_open_fireplaces == 0: + return + + estimated_cost = number_open_fireplaces * self.COST_OF_WORK + + # We recommend installing two mechanical ventilation systems + self.recommendation = [ + { + "parts": [], + "type": "sealing_open_fireplace", + "description": "Seal %s open fireplaces" % str(number_open_fireplaces), + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + "cost": estimated_cost, + } + ] From 761c1441c7a9aeedbc7edafa7c3a9fdc978d7879 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 19 Oct 2023 14:24:55 +1100 Subject: [PATCH 20/75] Fireplace recommendations tests --- backend/tests/test_sap_model_prep.py | 332 +++++++++++++++++++++------ 1 file changed, 259 insertions(+), 73 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 93567920..c27d05dc 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -15,7 +15,6 @@ import pickle with open("sap_change_dataset.pickle", "rb") as f: sap_change_dataset = pickle.load(f) -sap_change_dataset.columns.tolist() z = sap_change_dataset[(sap_change_dataset["NUMBER_OPEN_FIREPLACES_ENDING"] > 0) & ( sap_change_dataset["NUMBER_OPEN_FIREPLACES_STARTING"] > 0)].head(2).tail(1) z["UPRN"] @@ -24,81 +23,82 @@ z["SAP_ENDING"] z["NUMBER_OPEN_FIREPLACES_STARTING"] z["NUMBER_OPEN_FIREPLACES_ENDING"] - # 10002083298 # m -# search_from = sap_change_dataset[ -# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) -# ] -# search_from = search_from[ -# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & -# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & -# (search_from["MECHANICAL_VENTILATION_ENDING"] != search_from["MECHANICAL_VENTILATION_STARTING"]) & -# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & -# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) -# ] -# -# # Find a record where the only difference is cavity wall getting filled -# ending_cols = [c for c in search_from.columns if "_ENDING" in c] -# -# ignore = [ -# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", -# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -# ] -# -# ending_cols = [c for c in ending_cols if c not in ignore] -# -# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): -# -# same = True -# starting_cols = [] -# for c in ending_cols: -# -# starting_col = c.replace("_ENDING", "") -# if starting_col not in search_from.columns: -# starting_col = c.replace("_ENDING", "_STARTING") -# if starting_col not in search_from.columns: -# raise Exception("something went wrong") -# -# starting_cols.append(starting_col) -# -# # We want them to be different -# if c == "MECHANICAL_VENTILATION_ENDING": -# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): -# same = False -# break -# else: -# continue -# -# # We now check if the starting and ending values are the same -# if row[c] != row[starting_col]: -# same = False -# break -# -# if same: -# raise Exception("We found one!") -# -# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] -# -# import pandas as pd -# -# start = row[["SAP_STARTING"] + starting_cols] -# start.index = [c.replace("_STARTING", "") for c in start.index] -# end = row[["SAP_ENDING"] + ending_cols] -# end.index = [c.replace("_ENDING", "") for c in end.index] -# start["type"] = "starting" -# end["type"] = "ending" -# -# compare = pd.concat([start, end], axis=1) -# -# ending_lmk = "96b34dbe6cebdd7e648151e070047c8cd605c539851fc5a37e325903440081ab" -# starting_lmk = "dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247" -# -# client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "45 Shepperson Road", "postcode": "S6 4FG"}) -# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] +search_from = sap_change_dataset[ + (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) +] +search_from = search_from[ + (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & + (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & + (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & + (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & + (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) & + (search_from["NUMBER_OPEN_FIREPLACES_STARTING"] > 0) & + (search_from["NUMBER_OPEN_FIREPLACES_ENDING"] == 0) + ] + +# Find a record where the only difference is cavity wall getting filled +ending_cols = [c for c in search_from.columns if "_ENDING" in c] + +ignore = [ + "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", + "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +] + +ending_cols = [c for c in ending_cols if c not in ignore] + +for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): + + same = True + starting_cols = [] + for c in ending_cols: + + starting_col = c.replace("_ENDING", "") + if starting_col not in search_from.columns: + starting_col = c.replace("_ENDING", "_STARTING") + if starting_col not in search_from.columns: + raise Exception("something went wrong") + + starting_cols.append(starting_col) + + # We want them to be different + if c == "NUMBER_OPEN_FIREPLACES_ENDING": + if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): + same = False + break + else: + continue + + # We now check if the starting and ending values are the same + if row[c] != row[starting_col]: + same = False + break + + if same: + raise Exception("We found one!") + + fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] + + import pandas as pd + + start = row[["SAP_STARTING"] + starting_cols] + start.index = [c.replace("_STARTING", "") for c in start.index] + end = row[["SAP_ENDING"] + ending_cols] + end.index = [c.replace("_ENDING", "") for c in end.index] + start["type"] = "starting" + end["type"] = "ending" + + compare = pd.concat([start, end], axis=1) + +ending_lmk = "bab3983fa167717b8bb4a36ef395046d53937f9b880a45bcc751270d72e5de45" +starting_lmk = "736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef" + +client = EpcClient(auth_token=EPC_AUTH_TOKEN) +result = client.domestic.search(params={"address": "9 Glebe Road, Asfordby Hill", "postcode": "LE14 3QT"}) +starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] # with open( @@ -718,3 +718,189 @@ class TestSapModelPrep: continue assert test_record3[c].values[0] == row3[c] + + def test_fireplaces(self, cleaned, cleaning_data): + + starting_epc4 = { + 'low-energy-fixed-light-count': '', 'address': '9 Glebe Road, Asfordby Hill', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.4', 'heating-cost-potential': '501', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '70', + 'construction-age-band': 'England and Wales: 1930-1949', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', + 'environment-impact-potential': '76', 'glazed-type': 'double glazing, unknown install date', + 'heating-cost-current': '723', 'address3': '', + 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', + 'property-type': 'House', 'local-authority-label': 'Melton', + 'fixed-lighting-outlets-count': '14', 'energy-tariff': 'dual', + 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', + 'county': 'Leicestershire', 'postcode': 'LE14 3QT', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000909', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '5', + 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '153', + 'local-authority': 'E07000133', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '1', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', + 'inspection-date': '2022-06-27', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '46', + 'address1': '9 Glebe Road', 'heat-loss-corridor': '', 'flat-storey-count': '', + 'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Good', + 'total-floor-area': '87.0', 'building-reference-number': '10002396876', + 'environment-impact-current': '60', 'co2-emissions-current': '4.0', + 'roof-description': 'Pitched, 200 mm loft insulation', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Asfordby Hill', 'hot-water-env-eff': 'Good', + 'posttown': 'MELTON MOWBRAY', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Average', + 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 29% of fixed outlets', 'roof-env-eff': 'Good', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '15.0', 'lighting-cost-potential': '79', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', + 'lodgement-datetime': '2022-06-27 15:28:18', 'flat-top-storey': '', + 'current-energy-rating': 'D', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030539619', + 'current-energy-efficiency': '66', 'energy-consumption-current': '256', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '135', + 'lodgement-date': '2022-06-27', 'extension-count': '1', 'mainheatc-env-eff': 'Average', + 'lmk-key': '736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef', + 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', + 'potential-energy-efficiency': '78', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '29', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + } + + row4 = { + 'UPRN': '100030539619', 'RDSAP_CHANGE': 7, 'HEAT_DEMAND_CHANGE': -41, 'CARBON_CHANGE': -0.5, + 'SAP_STARTING': 66, 'SAP_ENDING': 73, 'HEAT_DEMAND_STARTING': 256, 'HEAT_DEMAND_ENDING': 215, + 'CARBON_STARTING': 4.0, 'CARBON_ENDING': 3.5, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Semi-Detached', + 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0, + 'FIXED_LIGHTING_OUTLETS_COUNT': 14.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1930-1949', + 'TRANSACTION_TYPE_STARTING': 'eco assessment', 'MECHANICAL_VENTILATION_STARTING': 'natural', + 'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_STARTING': 'dual', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 15.0, + 'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, + 'LOW_ENERGY_LIGHTING_STARTING': 29.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, + 'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 87.0, 'FLOOR_HEIGHT_STARTING': 2.4, + 'TRANSACTION_TYPE_ENDING': 'eco assessment', 'MECHANICAL_VENTILATION_ENDING': 'natural', + 'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_ENDING': 'dual', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 15.0, + 'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, + 'LOW_ENERGY_LIGHTING_ENDING': 29.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0, 'EXTENSION_COUNT_ENDING': 1.0, + 'TOTAL_FLOOR_AREA_ENDING': 87.0, 'FLOOR_HEIGHT_ENDING': 2.4, 'DAYS_TO_STARTING': 2887, + 'DAYS_TO_ENDING': 2960, 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, + 'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, + 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 1.7, 'is_park_home_ENDING': False, + 'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False, + 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.66, 'is_to_unheated_space': False, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, 'another_property_below': False, + 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.66, + 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.21, 'is_pitched': True, + 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'has_dwelling_above': False, 'roof_insulation_thickness': '200', 'roof_thermal_transmittance_ENDING': 0.21, + 'roof_insulation_thickness_ENDING': '200', 'heater_type': 'Unknown', 'system_type': 'from main system', + 'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', + 'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown', + 'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown', + 'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system', + 'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown', + 'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown', + 'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown', + 'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown', + 'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, + 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, + 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, + 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, + 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, + 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, + 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, + 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, + 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, + 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, + 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, + 'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False, + 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, + 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, + 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, + 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, + 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat', + 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', + 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, + 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown', + 'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown', + 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', + 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, + 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown', + 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'mains gas', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, + 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', + 'estimated_perimeter_STARTING': 37.54197650630557, 'estimated_perimeter_ENDING': 37.54197650630557 + } + + home4 = Property( + id=0, + postcode=starting_epc4["postcode"], + address1=starting_epc4["address1"], + epc_client=EpcClient(auth_token="notoken"), + data=starting_epc4 + ) + home4.get_components(cleaned) + + data_processor4 = DataProcessor(None, newdata=True) + data_processor4.insert_data(pd.DataFrame([home4.get_model_data()])) + + data_processor4.pre_process() + + starting_epc_data4 = data_processor4.get_component_features(suffix="_STARTING") + ending_epc_data4 = data_processor4.get_component_features(suffix="_ENDING") + fixed_data4 = data_processor4.get_fixed_features() + + ending_lodgement_date4 = '2022-09-08' + + ending_epc_data4["DAYS_TO_ENDING"] = data_processor4.calculate_days_to(ending_lodgement_date4) + + recommendation4 = { + "recommendation_id": 0, + "type": "sealing_open_fireplace" + } + + test_record4 = create_recommendation_scoring_data( + property=home4, + recommendation=recommendation4, + starting_epc_data=starting_epc_data4, + ending_epc_data=ending_epc_data4, + fixed_data=fixed_data4, + ) + test_record4 = pd.DataFrame([test_record4]) + + # Test the final cleaning: + test_record4 = DataProcessor.apply_averages_cleaning( + data_to_clean=test_record4, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + ).drop(columns=["LOCAL_AUTHORITY"]) + + test_record4 = DataProcessor.clean_missings_after_description_process( + test_record4, [ + c for c in test_record4.columns if + ("thermal_transmittance" in c) or ("insulation_thickness" in c) + ] + ) + + for c in test_record4.columns: + if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: + continue + + assert test_record4[c].values[0] == row4[c] From 52578287a7c3a1929bd2b455f5abe2efb714e894 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 19 Oct 2023 14:25:21 +1100 Subject: [PATCH 21/75] comment out temp code --- backend/tests/test_sap_model_prep.py | 168 +++++++++++++-------------- 1 file changed, 79 insertions(+), 89 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index c27d05dc..dc793cad 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -9,96 +9,86 @@ import pytest from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm + # Handy code for selecting testin data -import pickle - -with open("sap_change_dataset.pickle", "rb") as f: - sap_change_dataset = pickle.load(f) - -z = sap_change_dataset[(sap_change_dataset["NUMBER_OPEN_FIREPLACES_ENDING"] > 0) & ( - sap_change_dataset["NUMBER_OPEN_FIREPLACES_STARTING"] > 0)].head(2).tail(1) -z["UPRN"] -z["SAP_STARTING"] -z["SAP_ENDING"] -z["NUMBER_OPEN_FIREPLACES_STARTING"] -z["NUMBER_OPEN_FIREPLACES_ENDING"] - -# 10002083298 - -# m -search_from = sap_change_dataset[ - (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) -] -search_from = search_from[ - (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & - (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & - (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & - (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & - (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) & - (search_from["NUMBER_OPEN_FIREPLACES_STARTING"] > 0) & - (search_from["NUMBER_OPEN_FIREPLACES_ENDING"] == 0) - ] - -# Find a record where the only difference is cavity wall getting filled -ending_cols = [c for c in search_from.columns if "_ENDING" in c] - -ignore = [ - "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", - "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -] - -ending_cols = [c for c in ending_cols if c not in ignore] - -for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): - - same = True - starting_cols = [] - for c in ending_cols: - - starting_col = c.replace("_ENDING", "") - if starting_col not in search_from.columns: - starting_col = c.replace("_ENDING", "_STARTING") - if starting_col not in search_from.columns: - raise Exception("something went wrong") - - starting_cols.append(starting_col) - - # We want them to be different - if c == "NUMBER_OPEN_FIREPLACES_ENDING": - if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): - same = False - break - else: - continue - - # We now check if the starting and ending values are the same - if row[c] != row[starting_col]: - same = False - break - - if same: - raise Exception("We found one!") - - fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] - - import pandas as pd - - start = row[["SAP_STARTING"] + starting_cols] - start.index = [c.replace("_STARTING", "") for c in start.index] - end = row[["SAP_ENDING"] + ending_cols] - end.index = [c.replace("_ENDING", "") for c in end.index] - start["type"] = "starting" - end["type"] = "ending" - - compare = pd.concat([start, end], axis=1) - -ending_lmk = "bab3983fa167717b8bb4a36ef395046d53937f9b880a45bcc751270d72e5de45" -starting_lmk = "736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef" - -client = EpcClient(auth_token=EPC_AUTH_TOKEN) -result = client.domestic.search(params={"address": "9 Glebe Road, Asfordby Hill", "postcode": "LE14 3QT"}) -starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] +# import pickle +# +# with open("sap_change_dataset.pickle", "rb") as f: +# sap_change_dataset = pickle.load(f) +# +# search_from = sap_change_dataset[ +# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) +# ] +# search_from = search_from[ +# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & +# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & +# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & +# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & +# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) & +# (search_from["NUMBER_OPEN_FIREPLACES_STARTING"] > 0) & +# (search_from["NUMBER_OPEN_FIREPLACES_ENDING"] == 0) +# ] +# +# # Find a record where the only difference is cavity wall getting filled +# ending_cols = [c for c in search_from.columns if "_ENDING" in c] +# +# ignore = [ +# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", +# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +# ] +# +# ending_cols = [c for c in ending_cols if c not in ignore] +# +# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): +# +# same = True +# starting_cols = [] +# for c in ending_cols: +# +# starting_col = c.replace("_ENDING", "") +# if starting_col not in search_from.columns: +# starting_col = c.replace("_ENDING", "_STARTING") +# if starting_col not in search_from.columns: +# raise Exception("something went wrong") +# +# starting_cols.append(starting_col) +# +# # We want them to be different +# if c == "NUMBER_OPEN_FIREPLACES_ENDING": +# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): +# same = False +# break +# else: +# continue +# +# # We now check if the starting and ending values are the same +# if row[c] != row[starting_col]: +# same = False +# break +# +# if same: +# raise Exception("We found one!") +# +# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] +# +# import pandas as pd +# +# start = row[["SAP_STARTING"] + starting_cols] +# start.index = [c.replace("_STARTING", "") for c in start.index] +# end = row[["SAP_ENDING"] + ending_cols] +# end.index = [c.replace("_ENDING", "") for c in end.index] +# start["type"] = "starting" +# end["type"] = "ending" +# +# compare = pd.concat([start, end], axis=1) +# +# ending_lmk = "bab3983fa167717b8bb4a36ef395046d53937f9b880a45bcc751270d72e5de45" +# starting_lmk = "736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef" +# +# client = EpcClient(auth_token=EPC_AUTH_TOKEN) +# result = client.domestic.search(params={"address": "9 Glebe Road, Asfordby Hill", "postcode": "LE14 3QT"}) +# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] # with open( From d58a87af01f3aed03f5bc44be5e4ed217c41fb34 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 20 Oct 2023 11:46:38 +1100 Subject: [PATCH 22/75] Integrating new sap model process into backend --- backend/Property.py | 5 +++-- backend/app/plan/router.py | 25 ++++++++++++++++++++++--- etl/epc/DataProcessor.py | 14 ++++++++++---- etl/epc/settings.py | 3 ++- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 2b283e36..045b6220 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -4,6 +4,7 @@ import os import pandas as pd from etl.epc.DataProcessor import DataProcessor +from etl.epc.settings import POTENTIAL_COLUMNS, EFFICIENCY_FEATURES from etl.epc_clean.epc_attributes.all_cleaners import all_cleaner_map from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet @@ -603,7 +604,7 @@ class Property(Definitions): @staticmethod def _extract_component(component_data, component_rename_cols, component_drop_cols, rename_prefix=None): for k in component_rename_cols: - component_data[f"{rename_prefix}_{k}"] = component_data[k] + component_data[f"{rename_prefix}_{k}"] = component_data.get(k) component_data = { k: v for k, v in component_data.items() if k not in component_drop_cols + component_rename_cols @@ -640,7 +641,7 @@ class Property(Definitions): # We'll need to clean second heating second_heating = self.data["secondheat-description"] - epc_raw_columns = [ + epc_raw_columns = POTENTIAL_COLUMNS + EFFICIENCY_FEATURES + [ 'TRANSACTION_TYPE', 'ENERGY_TARIFF', 'PROPERTY_TYPE', diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8c8f309b..8a7591d8 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -125,14 +125,14 @@ async def trigger_plan(body: PlanTriggerRequest): # with open("input_properties.pickle", "rb") as f: # input_properties = pickle.load(f) # - # with open("cleaned.pickle", "rb") as f: - # cleaned = pickle.load(f) + # import pickle + # with open("new_sap_dataset.pickle", "rb") as f: + # new_sap_dataset = pickle.load(f) recommendations = {} recommendations_scoring_data = [] for p in input_properties: - property_recommendations = [] # Property recommendations @@ -234,6 +234,25 @@ async def trigger_plan(body: PlanTriggerRequest): ] ) + for c in new_sap_dataset.columns: + if c in ["UPRN", "RDSAP_CHANGE", "HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "SAP_STARTING"]: + continue + + if (new_sap_dataset[c].dtype.name in ["int64", "float64"]) & ( + recommendations_scoring_data[c].dtype.name in ["int64", "float64"] + ): + continue + + if c == "CONSTITUENCY": + if c not in recommendations_scoring_data: + raise Exception("wtf") + continue + + unique_vals = new_sap_dataset[c].unique() + scoring_unique_vals = recommendations_scoring_data[c].unique() + if not all(x in unique_vals for x in scoring_unique_vals): + raise Exception("") + sap_change_model_api = SAPChangeModelAPI(portfolio_id=body.portfolio_id, timestamp=created_at) file_location = sap_change_model_api.upload_scoring_data( df=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index cb3de9f4..3ef485b8 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -16,7 +16,9 @@ from etl.epc.settings import ( fill_na_map, STARTING_SUFFIX_COMPONENT_COLS, NO_SUFFIX_COMPONENT_COLS, - ENDING_SUFFIX_COMPONENT_COLS + ENDING_SUFFIX_COMPONENT_COLS, + POTENTIAL_COLUMNS, + EFFICIENCY_FEATURES, ) from recommendations.rdsap_tables import FLOOR_LEVEL_MAP @@ -203,6 +205,8 @@ class DataProcessor: # Final re-casting after data transformed and prepared coltypes = {k: v for k, v in COLUMNTYPES.items() if k in self.data.columns} if self.newdata else COLUMNTYPES + for k, v in coltypes.items(): + self.data[k] = self.data[k].astype(v) self.data = self.data.astype(coltypes) self.na_remapping() @@ -504,12 +508,14 @@ class DataProcessor: raise Exception("Suffix should be one of _STARTING or _ENDING") if suffix == "_STARTING": - starting_cols = self.data[STARTING_SUFFIX_COMPONENT_COLS].copy().add_suffix(suffix) - fixed_cols = self.data[NO_SUFFIX_COMPONENT_COLS].copy() + starting_cols = self.data[STARTING_SUFFIX_COMPONENT_COLS + EFFICIENCY_FEATURES].copy().add_suffix(suffix) + fixed_cols = self.data[NO_SUFFIX_COMPONENT_COLS + POTENTIAL_COLUMNS].copy() return pd.concat([starting_cols, fixed_cols], axis=1) - return self.data[ENDING_SUFFIX_COMPONENT_COLS].copy().add_suffix(suffix) + return self.data[ + ENDING_SUFFIX_COMPONENT_COLS + EFFICIENCY_FEATURES + ].copy().add_suffix(suffix) def get_fixed_features(self) -> pd.DataFrame: """ diff --git a/etl/epc/settings.py b/etl/epc/settings.py index 93b8929b..60c079a5 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -127,7 +127,6 @@ COMPONENT_FEATURES = CORE_COMPONENT_FEATURES + [ ] POTENTIAL_COLUMNS = [ - 'POTENTIAL_ENERGY_RATING', 'POTENTIAL_ENERGY_EFFICIENCY', 'ENVIRONMENT_IMPACT_POTENTIAL', 'ENERGY_CONSUMPTION_POTENTIAL', @@ -195,6 +194,8 @@ COLUMNTYPES = { 'MAINHEATCONT_DESCRIPTION': 'object', 'EXTENSION_COUNT': 'float64', 'LODGEMENT_DATE': 'object', + **dict(zip(EFFICIENCY_FEATURES, ['object', ] * len(EFFICIENCY_FEATURES))), + **dict(zip(POTENTIAL_COLUMNS, ['float64', ] * len(POTENTIAL_COLUMNS))) } # For modelling, we don't allow records with more than 100 SAP points From f6724b5ce9a79ba20a96ab0928b358514ab0e2e8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 20 Oct 2023 16:45:46 +1100 Subject: [PATCH 23/75] implementing new prediction process --- backend/app/plan/router.py | 26 ++++---------------------- backend/app/plan/utils.py | 2 ++ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8a7591d8..fdbf155d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -228,30 +228,12 @@ async def trigger_plan(body: PlanTriggerRequest): ).drop(columns=["LOCAL_AUTHORITY"]) recommendations_scoring_data = DataProcessor.clean_missings_after_description_process( - recommendations_scoring_data, [ - c for c in recommendations_scoring_data.columns if - ("thermal_transmittance" in c) or ("insulation_thickness" in c) - ] + recommendations_scoring_data, + ignore_cols=[c for c in recommendations_scoring_data.columns if ("thermal_transmittance" in c) or ( + "insulation_thickness" in c) or ("ENERGY_EFF" in c)] ) - for c in new_sap_dataset.columns: - if c in ["UPRN", "RDSAP_CHANGE", "HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "SAP_STARTING"]: - continue - - if (new_sap_dataset[c].dtype.name in ["int64", "float64"]) & ( - recommendations_scoring_data[c].dtype.name in ["int64", "float64"] - ): - continue - - if c == "CONSTITUENCY": - if c not in recommendations_scoring_data: - raise Exception("wtf") - continue - - unique_vals = new_sap_dataset[c].unique() - scoring_unique_vals = recommendations_scoring_data[c].unique() - if not all(x in unique_vals for x in scoring_unique_vals): - raise Exception("") + recommendations_scoring_data = DataProcessor.clean_efficiency_variables(recommendations_scoring_data) sap_change_model_api = SAPChangeModelAPI(portfolio_id=body.portfolio_id, timestamp=created_at) file_location = sap_change_model_api.upload_scoring_data( diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 2d9659e1..c06d9293 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -130,6 +130,7 @@ def create_recommendation_scoring_data( # insulation thickness scoring_dict["walls_thermal_transmittance_ENDING"] = recommendation["new_u_value"] scoring_dict["walls_insulation_thickness_ENDING"] = "above average" + scoring_dict["WALLS_ENERGY_EFF_ENDING"] = "Good" else: if scoring_dict["walls_thermal_transmittance_ENDING"] is None: scoring_dict["walls_thermal_transmittance_ENDING"] = get_wall_u_value( @@ -151,6 +152,7 @@ def create_recommendation_scoring_data( scoring_dict["floor_thermal_transmittance_ENDING"] = recommendation["new_u_value"] # We don't really see above average for this in the training data scoring_dict["floor_insulation_thickness_ENDING"] = "average" + scoring_dict["FLOOR_ENERGY_EFF_ENDING"] = "Good" else: if scoring_dict["floor_thermal_transmittance_ENDING"] is None: scoring_dict["floor_thermal_transmittance_ENDING"] = get_floor_u_value( From 13ceb4031d7125aaa7a0039638308276d3edab0f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 20 Oct 2023 18:51:49 +1100 Subject: [PATCH 24/75] implementing loft insulation wip --- recommendations/RoofRecommendations.py | 134 ++++++++++++++++++++++++ recommendations/WallRecommendations.py | 49 +++++---- recommendations/recommendation_utils.py | 17 +++ 3 files changed, 179 insertions(+), 21 deletions(-) create mode 100644 recommendations/RoofRecommendations.py diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py new file mode 100644 index 00000000..13cf958e --- /dev/null +++ b/recommendations/RoofRecommendations.py @@ -0,0 +1,134 @@ +import math +from backend import Property +from typing import List +from datatypes.enums import QuantityUnits +from recommendations.recommendation_utils import ( + get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, + update_lowest_selected_u_value, get_recommended_part +) + + +class RoofRecommendations: + # part L building regulations indicate that any rennovations on an existing property's roof should + # achieve a U-value of no higher than 0.16 + # This can be seen in table 4.3 in building regulations part L: + # https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/1133079 + # /Approved_Document_L__Conservation_of_fuel_and_power__Volume_1_Dwellings__2021_edition_incorporating_2023_amendments.pdf + BUILDING_REGULATIONS_PART_L_MAX_U_VALUE = 0.16 + + DIMINISHING_RETURNS_U_VALUE = 0.14 + + def __init__( + self, + property_instance: Property, + materials: List + ): + self.property = property_instance + # For audit purposes, when estimating u values we'll store it + self.estimated_u_value = None + + # Will contains a list of recommended measures + self.recommendations = [] + + self.materials = materials + + def recommend(self): + u_value = self.property.roof["thermal_transmittance"] + + insulation_thickness = self.property.walls["insulation_thickness"] + + # We check if the roof is already insulated and if so, we exit + if insulation_thickness in ["average", "above average"]: + return + + # If we have a u-value already, need to implement this + if u_value: + raise NotImplementedError("Implement me") + + u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) + + # With loft insulation, 100mm goes between the joists and the rest is rolled on top + # Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle + # from the base layer + materials = [ + { + 'id': 4, + 'type': 'loft_insulation', + 'description': 'Iso Spacesaver Mineral Wool insulation', + 'depths': [270, 300], + 'depth_unit': 'mm', + 'cost': [9, 10], + 'cost_unit': 'gbp_sq_meter', + 'r_value_per_mm': 0.022727272727272728, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.044, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': "https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m" + "-14-13m2/", + 'is_active': True + }, + ] + + self.materials = materials + + if self.property.roof["is_pitched"]: + # We recommend loft insulation + self.recommend_loft_insulation(u_value) + return + + def recommend_loft_insulation(self, u_value): + + """ + This method will recommend which insulation materials to use + :return: + """ + + loft_insulation_materials = [m for m in self.materials if m["type"] == "loft_insulation"] + + lowest_selected_u_value = None + recommendations = [] + for material in loft_insulation_materials: + + for depth, cost_per_unit in zip(material["depths"], material["cost"]): + + part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) + + _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) + new_u_value = math.ceil(new_u_value * 100.0) / 100.0 + + # If I have a lowest U value and my new u value is higher than that but lower than the + # diminishing returns threshold, it can be considered + + # If I have a lowest U value and my new u value is lower than the lowest value, it's + # further into the diminishing returns threshold and can shouldn't be + + if is_diminishing_returns( + recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE + ): + continue + + # We allow a small tolerance for error so we don't discount the recommendation entirely + if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + + estimated_cost = cost_per_unit * self.property.insulation_wall_area + + recommendations.append( + { + "parts": [ + get_recommended_part( + part=material, + selected_depth=depth, + quantity=self.property.insulation_wall_area, + quantity_unit=QuantityUnits.m2.value, + selected_total_cost=estimated_cost + ) + ], + "type": "roof_insulation", + "description": "TODO ", + "starting_u_value": u_value, + "new_u_value": new_u_value, + "sap_points": None, + "cost": estimated_cost, + } + ) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index ad2ca861..12085840 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -23,11 +23,17 @@ class WallRecommendations(Definitions): # part L building regulations indicate that any rennovations on an existing property's walls should # achieve a U-value of no higher than 0.3 + # This can be seen in table 4.3 in building regulations part L: + # https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/1133079 + # /Approved_Document_L__Conservation_of_fuel_and_power__Volume_1_Dwellings__2021_edition_incorporating_2023_amendments.pdf BUILDING_REGULATIONS_PART_L_MAX_U_VALUE = 0.3 # We don't recommend measures that are too low because it becomes expensive, therefore we aim to avoid # diminishing returns. This value should be verified with Osmosis (TODO) DIMINISHING_RETURNS_U_VALUE = 0.25 + # Building regulations part L also indicates that cavity wall insulation should result in 0.55 u-value + BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE = 0.55 + # Part L regulations indicate that any new build should have walls that achieve a u-value of no higher # than 0.18. BUILDING_REGULATIONS_PART_L_NEW_BUILD_MAX_U_VALUE = 0.18 @@ -167,29 +173,30 @@ class WallRecommendations(Definitions): ): continue - lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = part["cost"] * self.property.insulation_wall_area + estimated_cost = part["cost"] * self.property.insulation_wall_area - recommendations.append( - { - "parts": [ - get_recommended_part( - part=part, - selected_depth=None, - quantity=self.property.insulation_wall_area, - quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost - ) - ], - "type": "wall_insulation", - "description": f"Fill cavity with {part['description']}", - "starting_u_value": u_value, - "new_u_value": new_u_value, - "sap_points": None, - "cost": estimated_cost, - } - ) + recommendations.append( + { + "parts": [ + get_recommended_part( + part=part, + selected_depth=None, + quantity=self.property.insulation_wall_area, + quantity_unit=QuantityUnits.m2.value, + selected_total_cost=estimated_cost + ) + ], + "type": "wall_insulation", + "description": f"Fill cavity with {part['description']}", + "starting_u_value": u_value, + "new_u_value": new_u_value, + "sap_points": None, + "cost": estimated_cost, + } + ) self.recommendations = recommendations diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index cd7bb3f8..4f0813ab 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -517,3 +517,20 @@ def estimate_wall_area(num_floors, floor_height, perimeter): total_wall_area = wall_area_one_floor * num_floors return total_wall_area + + +def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): + """ + # Calculate R-value (thermal resistance) using the formula: R = thickness / thermal_conductivity + # Note: The thickness should be converted to meters for the units to be consistent. + :param thickness_mm: + :param thermal_conductivity_w_mK: + :return: + """ + + r_value_m2k_w = (thickness_mm / 1000) / thermal_conductivity_w_mK + + # Calculate R-value per mm + r_value_per_mm = r_value_m2k_w / thickness_mm + + return r_value_per_mm From 67e6a555f0611a1d876e726868b50736cf68dd40 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 20 Oct 2023 19:01:30 +1100 Subject: [PATCH 25/75] almost completed loft insulation --- recommendations/RoofRecommendations.py | 40 +++++++++----------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 13cf958e..5a2a6bcf 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -47,35 +47,17 @@ class RoofRecommendations: u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) - # With loft insulation, 100mm goes between the joists and the rest is rolled on top - # Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle - # from the base layer - materials = [ - { - 'id': 4, - 'type': 'loft_insulation', - 'description': 'Iso Spacesaver Mineral Wool insulation', - 'depths': [270, 300], - 'depth_unit': 'mm', - 'cost': [9, 10], - 'cost_unit': 'gbp_sq_meter', - 'r_value_per_mm': 0.022727272727272728, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': "https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m" - "-14-13m2/", - 'is_active': True - }, - ] - - self.materials = materials - if self.property.roof["is_pitched"]: # We recommend loft insulation self.recommend_loft_insulation(u_value) return + raise NotImplementedError("Implement me") + + @staticmethod + def make_loft_insulation_description(material, depth): + return f"Install {depth}{material['depth_unit']} of {material['description']}" + def recommend_loft_insulation(self, u_value): """ @@ -83,6 +65,10 @@ class RoofRecommendations: :return: """ + # With loft insulation, 100mm goes between the joists and the rest is rolled on top + # Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle + # from the base layer + loft_insulation_materials = [m for m in self.materials if m["type"] == "loft_insulation"] lowest_selected_u_value = None @@ -111,7 +97,7 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = cost_per_unit * self.property.insulation_wall_area + estimated_cost = cost_per_unit * self.property.floor_area recommendations.append( { @@ -125,10 +111,12 @@ class RoofRecommendations: ) ], "type": "roof_insulation", - "description": "TODO ", + "description": self.make_loft_insulation_description(material, depth), "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, "cost": estimated_cost, } ) + + self.recommendations = recommendations From 193e0137646a9057831a191485dc0758948660e0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 20 Oct 2023 19:21:42 +1100 Subject: [PATCH 26/75] added roof insulation to backend - almost complete --- backend/app/db/models/materials.py | 1 + backend/app/plan/router.py | 22 +++++++------- backend/app/plan/utils.py | 46 ++++++++++++++++++------------ 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index cf6dd971..1dc47276 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -14,6 +14,7 @@ class MaterialType(enum.Enum): internal_wall_insulation = "internal_wall_insulation" cavity_wall_insulation = "cavity_wall_insulation" mechanical_ventilation = "mechanical_ventilation" + loft_insulation = "loft_insulation" class DepthUnit(enum.Enum): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fdbf155d..d5555d0b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,6 +30,7 @@ from backend.Property import Property from etl.epc.DataProcessor import DataProcessor from etl.epc.settings import COLUMNS_TO_MERGE_ON from recommendations.FloorRecommendations import FloorRecommendations +from recommendations.RoofRecommendations import RoofRecommendations from recommendations.VentilationRecommendations import VentilationRecommendations from recommendations.FireplaceRecommendations import FireplaceRecommendations from recommendations.optimiser.CostOptimiser import CostOptimiser @@ -139,10 +140,7 @@ async def trigger_plan(body: PlanTriggerRequest): p.get_components(cleaned) # Floor recommendations - floor_recommender = FloorRecommendations( - property_instance=p, - materials=materials_by_type["floor"], - ) + floor_recommender = FloorRecommendations(property_instance=p, materials=materials_by_type["floor"]) floor_recommender.recommend() if floor_recommender.recommendations: @@ -150,15 +148,19 @@ async def trigger_plan(body: PlanTriggerRequest): # Wall recommendations - wall_recomender = WallRecommendations( - property_instance=p, - materials=materials_by_type["walls"] - ) + wall_recomender = WallRecommendations(property_instance=p, materials=materials_by_type["walls"]) wall_recomender.recommend() if wall_recomender.recommendations: property_recommendations.append(wall_recomender.recommendations) + # Roof recommendations + roof_recommender = RoofRecommendations(property_instance=p, materials=materials_by_type["roof"]) + roof_recommender.recommend() + + if roof_recommender.recommendations: + property_recommendations.append(roof_recommender.recommendations) + # Ventilation recommendations ventilation_recomender = VentilationRecommendations( property_instance=p, @@ -170,9 +172,7 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations.append(ventilation_recomender.recommendation) # Fireplace sealing recommendations - fireplace_recommender = FireplaceRecommendations( - property_instance=p - ) + fireplace_recommender = FireplaceRecommendations(property_instance=p) fireplace_recommender.recommend() if fireplace_recommender.recommendation: diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index c06d9293..5a761b2d 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -17,6 +17,7 @@ def filter_materials(materials): "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], "floor": ["suspended_floor_insulation", "solid_floor_insulation"], "ventilation": ["mechanical_ventilation"], + "roof": ["loft_insulation"] } materials = [row2dict(material) for material in materials] @@ -145,7 +146,6 @@ def create_recommendation_scoring_data( # Update description to indicate it's insulate if recommendation["type"] == "floor_insulation": - if len(recommendation["parts"]) > 1: raise NotImplementedError("Have more than 1 floor insulation part - handle this case") @@ -167,6 +167,33 @@ def create_recommendation_scoring_data( if scoring_dict["floor_insulation_thickness_ENDING"] is None: scoring_dict["floor_insulation_thickness_ENDING"] = "none" + if recommendation["type"] == "roof_insulation": + scoring_dict["roof_thermal_transmittance_ENDING"] = recommendation["new_u_value"] + + parts = recommendation["parts"] + if len(parts) != 1: + raise ValueError("More than one part for roof insulation - investiage me") + + scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) + scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" + else: + # Fill missing roof u-values - this fill is not based on recommended upgrades + if scoring_dict["roof_thermal_transmittance_ENDING"] is None: + scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value( + insulation_thickness=property.roof["insulation_thickness"], + has_dwelling_above=property.roof["has_dwelling_above"], + is_loft=property.roof["is_loft"], + is_roof_room=property.roof["is_roof_room"], + is_thatched=property.roof["is_thatched"], + age_band=property.age_band, + is_flat=property.roof["is_flat"], + is_pitched=property.roof["is_pitched"], + is_at_rafters=property.roof["is_at_rafters"], + ) + + if scoring_dict["roof_insulation_thickness_ENDING"] is None: + scoring_dict["roof_insulation_thickness_ENDING"] = "none" + if recommendation["type"] == "mechanical_ventilation": scoring_dict["MECHANICAL_VENTILATION_ENDING"] = 'mechanical, extract only' @@ -178,21 +205,4 @@ def create_recommendation_scoring_data( ]: raise NotImplementedError("Implement me") - # Fill missing roof u-values - this fill is not based on recommended upgrades - if scoring_dict["roof_thermal_transmittance_ENDING"] is None: - scoring_dict["roof_thermal_transmittance_ENDING"] = get_roof_u_value( - insulation_thickness=property.roof["insulation_thickness"], - has_dwelling_above=property.roof["has_dwelling_above"], - is_loft=property.roof["is_loft"], - is_roof_room=property.roof["is_roof_room"], - is_thatched=property.roof["is_thatched"], - age_band=property.age_band, - is_flat=property.roof["is_flat"], - is_pitched=property.roof["is_pitched"], - is_at_rafters=property.roof["is_at_rafters"], - ) - - if scoring_dict["roof_insulation_thickness_ENDING"] is None: - scoring_dict["roof_insulation_thickness_ENDING"] = "none" - return scoring_dict From 9346b82c2e5022640beff6461b6da1f450a75841 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 21 Oct 2023 20:18:18 +1100 Subject: [PATCH 27/75] allow roof_insulation in create_recommendation_scoring_data --- backend/app/plan/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 5a761b2d..e056c61c 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -201,7 +201,7 @@ def create_recommendation_scoring_data( scoring_dict["NUMBER_OPEN_FIREPLACES_ENDING"] = 0 if recommendation["type"] not in [ - "wall_insulation", "floor_insulation", "mechanical_ventilation", "sealing_open_fireplace" + "wall_insulation", "floor_insulation", "roof_insulation", "mechanical_ventilation", "sealing_open_fireplace", ]: raise NotImplementedError("Implement me") From 57c8788a373f0e96bc87a6b742a183020b614e6a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 06:23:05 +1100 Subject: [PATCH 28/75] basic unit tests for fireplace recommendations --- backend/app/plan/router.py | 10 ++++ recommendations/RoofRecommendations.py | 11 ++-- recommendations/recommendation_utils.py | 26 +++++++++ .../tests/test_fireplace_recommendations.py | 58 +++++++++++++++++++ .../tests/test_recommendation_utils.py | 11 ++++ .../tests/test_roof_recommendations.py | 39 +++++++++++++ 6 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 recommendations/tests/test_fireplace_recommendations.py create mode 100644 recommendations/tests/test_roof_recommendations.py diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d5555d0b..6171ddbf 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -130,6 +130,16 @@ async def trigger_plan(body: PlanTriggerRequest): # with open("new_sap_dataset.pickle", "rb") as f: # new_sap_dataset = pickle.load(f) + # import pickle + # with open("cleaned.pickle", "rb") as f: + # cleaned = pickle.dump(f) + + # with open("sap_dataset.pickle", "rb") as f: + # sap_dataset = pickle.load(f) + + # with open("materials_by_type", "rb") as f: + # materials_by_type = pickle.load(f) + recommendations = {} recommendations_scoring_data = [] diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 5a2a6bcf..06eab302 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -1,10 +1,10 @@ import math -from backend import Property +from backend.Property import Property from typing import List from datatypes.enums import QuantityUnits from recommendations.recommendation_utils import ( get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, - update_lowest_selected_u_value, get_recommended_part + update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric ) @@ -35,10 +35,13 @@ class RoofRecommendations: def recommend(self): u_value = self.property.roof["thermal_transmittance"] - insulation_thickness = self.property.walls["insulation_thickness"] + insulation_thickness = convert_thickness_to_numeric(self.property.roof["insulation_thickness"]) # We check if the roof is already insulated and if so, we exit - if insulation_thickness in ["average", "above average"]: + + # Building regulations part L recommend installing at least 270mm of insulation, however generally we + # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation + if insulation_thickness < 270: return # If we have a u-value already, need to implement this diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 4f0813ab..67550fc8 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -534,3 +534,29 @@ def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): r_value_per_mm = r_value_m2k_w / thickness_mm return r_value_per_mm + + +def convert_thickness_to_numeric(string_thickness): + """ + Roof insulation thickness could be a string like "None", "300mm+" or a numeric string. + This function will convert these strings to a number for easy usage + :param string_thickness: string measure of insulation thickness + :return: integer measure of insulation thickness + """ + + lookup = { + "none": 0, + "below average": 50, + "average": 100, + "above average": 270 + } + + mapped = lookup.get(string_thickness) + + if mapped is not None: + return mapped + + if "+" in string_thickness: + return int(string_thickness.replace("+", "")) + + return int(string_thickness) diff --git a/recommendations/tests/test_fireplace_recommendations.py b/recommendations/tests/test_fireplace_recommendations.py new file mode 100644 index 00000000..a1e0c1c6 --- /dev/null +++ b/recommendations/tests/test_fireplace_recommendations.py @@ -0,0 +1,58 @@ +from backend.Property import Property +from unittest.mock import Mock +from recommendations.FireplaceRecommendations import FireplaceRecommendations + + +class TestFirepaceRecommendations: + + def test_no_fireplaces(self): + property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance.data = { + "number-open-fireplaces": 0 + } + + recommender = FireplaceRecommendations( + property_instance=property_instance + ) + + assert recommender.recommendation is None + + recommender.recommend() + + assert recommender.recommendation is None + + def test_one_fireplace(self): + property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance.data = { + "number-open-fireplaces": 1 + } + + recommender = FireplaceRecommendations( + property_instance=property_instance + ) + + assert recommender.recommendation is None + + recommender.recommend() + + assert recommender.recommendation + assert recommender.recommendation[0]["type"] == "sealing_open_fireplace" + assert recommender.recommendation[0]["cost"] == 300 + + def test_multiple_fireplaces(self): + property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance.data = { + "number-open-fireplaces": 3 + } + + recommender = FireplaceRecommendations( + property_instance=property_instance + ) + + assert recommender.recommendation is None + + recommender.recommend() + + assert recommender.recommendation + assert recommender.recommendation[0]["type"] == "sealing_open_fireplace" + assert recommender.recommendation[0]["cost"] == 900 diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index eb1a5024..dc98d946 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -277,6 +277,17 @@ class TestRecommendationUtils: insulation_thickness=None, ) + def test_convert_thickness_to_numeric(self): + + assert recommendation_utils.convert_thickness_to_numeric("none") == 0 + assert recommendation_utils.convert_thickness_to_numeric("below average") == 50 + assert recommendation_utils.convert_thickness_to_numeric("average") == 100 + assert recommendation_utils.convert_thickness_to_numeric("above average") == 270 + + assert recommendation_utils.convert_thickness_to_numeric("300+") == 300 + assert recommendation_utils.convert_thickness_to_numeric("400+") == 400 + assert recommendation_utils.convert_thickness_to_numeric("270") == 270 + def test_estimate_perimeter_regular_inputs(): assert math.isclose( diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py new file mode 100644 index 00000000..b5507421 --- /dev/null +++ b/recommendations/tests/test_roof_recommendations.py @@ -0,0 +1,39 @@ +from backend.Property import Property +from unittest.mock import Mock +from recommendations.RoofRecommendations import RoofRecommendations + +loft_insulation_materials = [ + { + 'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation', + 'depths': [270, 300], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter', + 'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/', + 'is_active': True + } +] + + +class TestRoofRecommendations: + + def test_loft_insulation_recommendation_no_insulation(self): + property_instance = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance.age_band = "F" + property_instance.floor_area = 100 + property_instance.roof = { + 'original_description': 'Pitched, no insulation (assumed)', + 'clean_description': 'Pitched, no insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': 'none', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender = RoofRecommendations(property_instance=property_instance, materials=loft_insulation_materials) + + assert not roof_recommender.recommendations + + roof_recommender.recommend() + + assert len(roof_recommender.recommendations) From 0a549e6916963a42bba2bfc7410f90e02c9df0fd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 06:56:29 +1100 Subject: [PATCH 29/75] wrote complete loft insulation recommendations --- backend/app/plan/router.py | 2 +- recommendations/RoofRecommendations.py | 15 +- .../tests/test_roof_recommendations.py | 155 ++++++++++++++++++ 3 files changed, 168 insertions(+), 4 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6171ddbf..59e6cc32 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -121,7 +121,7 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - + # import pickle # with open("input_properties.pickle", "rb") as f: # input_properties = pickle.load(f) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 06eab302..d364dea9 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -18,6 +18,9 @@ class RoofRecommendations: DIMINISHING_RETURNS_U_VALUE = 0.14 + # It is recommended that lofts should have at least 270mm of insulation + MINIMUM_LOFT_ISULATION_MM = 270 + def __init__( self, property_instance: Property, @@ -41,7 +44,7 @@ class RoofRecommendations: # Building regulations part L recommend installing at least 270mm of insulation, however generally we # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation - if insulation_thickness < 270: + if insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM: return # If we have a u-value already, need to implement this @@ -52,7 +55,7 @@ class RoofRecommendations: if self.property.roof["is_pitched"]: # We recommend loft insulation - self.recommend_loft_insulation(u_value) + self.recommend_loft_insulation(u_value, insulation_thickness) return raise NotImplementedError("Implement me") @@ -61,10 +64,12 @@ class RoofRecommendations: def make_loft_insulation_description(material, depth): return f"Install {depth}{material['depth_unit']} of {material['description']}" - def recommend_loft_insulation(self, u_value): + def recommend_loft_insulation(self, u_value, insulation_thickness): """ This method will recommend which insulation materials to use + :param u_value: U-value of the roof before any retrofit measures have been installed + :param insulation_thickness: Existing Insulation thickness of the loft :return: """ @@ -79,6 +84,10 @@ class RoofRecommendations: for material in loft_insulation_materials: for depth, cost_per_unit in zip(material["depths"], material["cost"]): + # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the + # loft is already partially insulated + if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: + continue part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index b5507421..8243a35b 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -13,6 +13,28 @@ loft_insulation_materials = [ } ] +loft_insulation_materials_50mm_existing = [ + { + 'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation', + 'depths': [220, 210], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter', + 'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/', + 'is_active': True + } +] + +loft_insulation_materials_150mm_existing = [ + { + 'id': 18, 'type': 'loft_insulation', 'description': 'Iso Spacesaver Mineral Wool insulation', + 'depths': [130, 119], 'depth_unit': 'mm', 'cost': [9, 10], 'cost_unit': 'gbp_sq_meter', + 'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-100mm-x-1160mm-x-12-18m-14-13m2/', + 'is_active': True + } +] + class TestRoofRecommendations: @@ -37,3 +59,136 @@ class TestRoofRecommendations: roof_recommender.recommend() assert len(roof_recommender.recommendations) + + def test_loft_insulation_recommendation_50mm_insulation(self): + property_instance2 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance2.age_band = "F" + property_instance2.floor_area = 100 + property_instance2.roof = { + 'original_description': 'Pitched, 50mm loft insulation (assumed)', + 'clean_description': 'Pitched, 50mm loft insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': '50', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender2 = RoofRecommendations( + property_instance=property_instance2, materials=loft_insulation_materials + ) + + assert not roof_recommender2.recommendations + + roof_recommender2.recommend() + + assert len(roof_recommender2.recommendations) == 1 + + assert roof_recommender2.recommendations[0]["cost"] == 900 + assert roof_recommender2.recommendations[0]["new_u_value"] == 0.14 + assert roof_recommender2.recommendations[0]["starting_u_value"] == 0.68 + + property_instance3 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance3.age_band = "F" + property_instance3.floor_area = 100 + property_instance3.roof = { + 'original_description': 'Pitched, 50mm loft insulation (assumed)', + 'clean_description': 'Pitched, 50mm loft insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': '50', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender3 = RoofRecommendations( + property_instance=property_instance3, materials=loft_insulation_materials_50mm_existing + ) + + assert not roof_recommender3.recommendations + + roof_recommender3.recommend() + + # The 220mm insulation should be selected, not the 210 + assert roof_recommender3.recommendations + assert len(roof_recommender3.recommendations) == 1 + assert roof_recommender3.recommendations[0]["parts"][0]["depths"] == [220] + + def test_loft_insulation_recommendation_150mm_insulation(self): + property_instance4 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance4.age_band = "F" + property_instance4.floor_area = 100 + property_instance4.roof = { + 'original_description': 'Pitched, 150mm loft insulation (assumed)', + 'clean_description': 'Pitched, 150mm loft insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': '150', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender4 = RoofRecommendations( + property_instance=property_instance4, materials=loft_insulation_materials + ) + + assert not roof_recommender4.recommendations + + roof_recommender4.recommend() + + assert len(roof_recommender4.recommendations) == 1 + + assert roof_recommender4.recommendations[0]["cost"] == 900 + assert roof_recommender4.recommendations[0]["new_u_value"] == 0.11 + assert roof_recommender4.recommendations[0]["starting_u_value"] == 0.3 + + property_instance5 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance5.age_band = "F" + property_instance5.floor_area = 100 + property_instance5.roof = { + 'original_description': 'Pitched, 150mm loft insulation (assumed)', + 'clean_description': 'Pitched, 150mm loft insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': '150', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender5 = RoofRecommendations( + property_instance=property_instance5, materials=loft_insulation_materials_150mm_existing + ) + + assert not roof_recommender5.recommendations + + roof_recommender5.recommend() + + # The 130mm insulation should be selected, not the 110 + assert roof_recommender5.recommendations + assert len(roof_recommender5.recommendations) == 1 + assert roof_recommender5.recommendations[0]["parts"][0]["depths"] == [130] + + def test_loft_insulation_recommendation_270mm_insulation(self): + # We shouldn't recommend anything in this case + property_instance6 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance6.age_band = "F" + property_instance6.floor_area = 100 + property_instance6.roof = { + 'original_description': 'Pitched, 270mm loft insulation (assumed)', + 'clean_description': 'Pitched, 270mm loft insulation', + 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': '270', 'roof_thermal_transmittance': None, 'roof_insulation_thickness': 'none' + } + + roof_recommender6 = RoofRecommendations( + property_instance=property_instance6, materials=loft_insulation_materials + ) + + assert not roof_recommender6.recommendations + + roof_recommender6.recommend() + + assert len(roof_recommender6.recommendations) == 0 From 1228c680a6b9b9905a8d9e4faefed1ef521d7c73 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 15:19:18 +0800 Subject: [PATCH 30/75] implemented unheated space floor u-value --- backend/Property.py | 21 +++++++++++++-- backend/app/plan/router.py | 8 +++--- etl/epc/property_change_app.py | 2 -- recommendations/FloorRecommendations.py | 2 ++ recommendations/rdsap_tables.py | 28 ++++++++++++++++++++ recommendations/recommendation_utils.py | 34 ++++++++++++++++++++++++- 6 files changed, 86 insertions(+), 9 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 045b6220..b0c6b083 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -597,9 +597,26 @@ class Property(Definitions): def set_floor_type(self): """ This method sets the floor type of the property, which is used for calculating u-values - :return: + + Section 5.6 of the BRE indicates that + "to simplify data collection no distinction is made in terms of U-value between an exposed floor (to + outside air below) and a semi-exposed floor (to an enclosed but unheated space below) + and the U-values in Table S12 are used. + + Therefore, we treat the exposed floor and suspended floor as the same type of floor, which is used for + calculating u-values """ - self.floor_type = "suspended" if self.floor["is_suspended"] else "solid" + + if self.floor["is_suspended"] | self.floor["another_property_below"]: + self.floor_type = "suspended" + elif self.floor["is_solid"]: + self.floor_type = "solid" + elif self.floor["is_to_unheated_space"] | self.floor["is_to_external_air"]: + self.floor_type = "exposed_floor" + elif self.floor["thermal_transmittance"] is not None: + self.floor_type = "solid" + else: + raise NotImplementedError("Implement this floor type") @staticmethod def _extract_component(component_data, component_rename_cols, component_drop_cols, rename_prefix=None): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 59e6cc32..d3ea3f83 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -121,7 +121,7 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - + # import pickle # with open("input_properties.pickle", "rb") as f: # input_properties = pickle.load(f) @@ -132,7 +132,7 @@ async def trigger_plan(body: PlanTriggerRequest): # import pickle # with open("cleaned.pickle", "rb") as f: - # cleaned = pickle.dump(f) + # cleaned = pickle.load(f) # with open("sap_dataset.pickle", "rb") as f: # sap_dataset = pickle.load(f) @@ -144,11 +144,11 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations_scoring_data = [] for p in input_properties: - property_recommendations = [] - # Property recommendations p.get_components(cleaned) + property_recommendations = [] + # Floor recommendations floor_recommender = FloorRecommendations(property_instance=p, materials=materials_by_type["floor"]) floor_recommender.recommend() diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 435b668d..4f49f6da 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -415,13 +415,11 @@ def app(): all_equal_rows = [] for directory in tqdm(directories): - filepath = directory / "certificates.csv" data_processor = DataProcessor(filepath=filepath) df = data_processor.pre_process() - df[df["WALLS_DESCRIPTION"].str.contains("Cavity")]["WALLS_DESCRIPTION"].unique() cleaning_averages = data_processor.make_cleaning_averages() diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 35e34648..a20a4fe1 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -97,6 +97,8 @@ class FloorRecommendations(Definitions): # Given the U-value, we recommend solid floor insulation options which are usually solid foam self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts) + raise NotImplementedError("Implement me!") + @staticmethod def _make_floor_description(part, depth): return f"Install {depth}{part['depth_unit']} {part['description']} insulation" diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index 0ce139ab..e396f727 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -463,6 +463,34 @@ s11_list = [ table_s11 = pd.DataFrame(s11_list) +######################################################################################################################## +# Table s12 is used for assigning the u-values of floors to unheated spaces or external air +# which can be found on page 26 of the BRE document, section 5.6 +# https://bregroup.com/wp-content/uploads/2019/09/RdSAP_2012_9.94-20-09-2019.pdf +# +# the insulation_{thickness} fields indicate the u-value at that insulation thickness +######################################################################################################################## + +s12_list = [ + {"age_band": "A", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "B", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "C", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "D", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "E", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "F", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "G", "insulation_0": 1.2, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + + {"age_band": "H", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + {"age_band": "I", "insulation_0": 0.51, "insulation_50": 0.5, "insulation_100": 0.3, "insulation_150": 0.22}, + + {"age_band": "J", "insulation_0": 0.25, "insulation_50": 0.25, "insulation_100": 0.25, "insulation_150": 0.22}, + + {"age_band": "K", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22}, + {"age_band": "L", "insulation_0": 0.22, "insulation_50": 0.22, "insulation_100": 0.22, "insulation_150": 0.22}, +] + +table_s12 = pd.DataFrame(s12_list) + ######################################################################################################################## # diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 67550fc8..8e67a384 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -5,7 +5,7 @@ import pandas as pd from recommendations.rdsap_tables import ( epc_wall_description_map, wall_uvalues_df, default_wall_thickness, table_s9 as s9, table_s10 as s10, - table_s11 as s11 + table_s11 as s11, table_s12 as s12 ) from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION, PARTIAL_CAVITY_DESCRIPTIONS @@ -340,6 +340,32 @@ def estimate_perimeter(floor_area, num_rooms): return perimeter +def get_exposed_floor_uvalue(insulation_thickness_str, age_band): + """ + We implement the methodology as defined in section 5.6 and table S12 of the RdSAP document + :param insulation_thickness_str: + :return: + """ + + if age_band in ["A", "B", "C", "D", "E", "F", "G", "H", "I"]: + # As directed by the documentation, if the insulation thickness is not known, we assume it's + # 50mm for these age bands + if insulation_thickness_str in ["below_average", "average", "above_average"]: + insulation_thickness = 50 + elif insulation_thickness_str in ["none", None]: + insulation_thickness = 0 + elif insulation_thickness_str in ["below_average"]: + insulation_thickness = 50 + elif insulation_thickness_str == "average": + insulation_thickness = 100 + elif insulation_thickness_str == "above_average": + insulation_thickness = 150 + else: + insulation_thickness = int(insulation_thickness_str.replace("mm", "")) + + return s12[s12["age_band"] == age_band][f"insulation_{insulation_thickness}"].values[0] + + def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None): """ Estimate the u-value of a suspended floor, based on RdSap methodology @@ -372,6 +398,12 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati 0.701 """ + if floor_type == "exposed_floor": + # In this case, we extract the u-value from table s12 + # See section 5.6 of the RdSAP document for more details + # https://bregroup.com/wp-content/uploads/2019/09/RdSAP_2012_9.94-20-09-2019.pdf + return get_exposed_floor_uvalue(insulation_thickness, age_band) + # Cleans our regularly inputted insulation thickness for usage in this function insulation_thickness = extract_insulation_thickness(insulation_thickness) From 448c6a377239d33feb1cc8e732eaf10cf5f787ef Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 16:02:08 +0800 Subject: [PATCH 31/75] added exposed floor insulation --- backend/app/plan/utils.py | 2 +- recommendations/FloorRecommendations.py | 14 ++++++++++---- recommendations/recommendation_utils.py | 12 +++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index e056c61c..71a61be1 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -15,7 +15,7 @@ def filter_materials(materials): mapping = { "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], - "floor": ["suspended_floor_insulation", "solid_floor_insulation"], + "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"], "ventilation": ["mechanical_ventilation"], "roof": ["loft_insulation"] } diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index a20a4fe1..2524d25f 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -45,10 +45,13 @@ class FloorRecommendations(Definitions): part for part in self.materials if part["type"] == "solid_floor_insulation" ] + self.exposed_floor_insulation_parts = [ + part for part in self.materials if part["type"] == "exposed_floor_insulation" + ] + def recommend(self): u_value = self.property.floor["thermal_transmittance"] - is_suspended = self.property.floor["is_suspended"] - is_solid = self.property.floor["is_solid"] + floor_level = ( FLOOR_LEVEL_MAP[self.property.data["floor-level"]] if self.property.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None @@ -89,14 +92,17 @@ class FloorRecommendations(Definitions): ) self.estimated_u_value = u_value - if is_suspended: + if self.property.floor["is_suspended"]: # Given the U-value, we recommend underfloor insulation self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts) - if is_solid: + if self.property.floor["is_solid"]: # Given the U-value, we recommend solid floor insulation options which are usually solid foam self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts) + if self.property.floor["is_to_unheated_space"] or self.property.floor["is_to_external_air"]: + self.recommend_floor_insulation(u_value=u_value, parts=self.exposed_floor_insulation_parts) + raise NotImplementedError("Implement me!") @staticmethod diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 8e67a384..74910e07 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -347,11 +347,13 @@ def get_exposed_floor_uvalue(insulation_thickness_str, age_band): :return: """ - if age_band in ["A", "B", "C", "D", "E", "F", "G", "H", "I"]: - # As directed by the documentation, if the insulation thickness is not known, we assume it's - # 50mm for these age bands - if insulation_thickness_str in ["below_average", "average", "above_average"]: - insulation_thickness = 50 + unknown_insulation_age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + # As directed by the documentation, if the insulation thickness is not known, we assume it's + # 50mm for these age bands + if insulation_thickness_str in ["below_average", "average", "above_average"] and ( + age_band in unknown_insulation_age_bands + ): + insulation_thickness = 50 elif insulation_thickness_str in ["none", None]: insulation_thickness = 0 elif insulation_thickness_str in ["below_average"]: From 0c5ff1153c21a3d1540f821380e75f223ec4fc45 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 16:27:38 +0800 Subject: [PATCH 32/75] unit tests for unheated space floor insulation --- recommendations/FloorRecommendations.py | 14 +- recommendations/recommendation_utils.py | 6 +- .../tests/test_floor_recommendations.py | 142 +++++++++++++++++- 3 files changed, 155 insertions(+), 7 deletions(-) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 2524d25f..a78d984e 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -58,6 +58,7 @@ class FloorRecommendations(Definitions): ) property_type = self.property.data["property-type"] + floor_area = self.property.floor_area / self.property.number_of_storeys year_built = self.property.year_built if self.property.floor["another_property_below"] | (self.property.floor["insulation_thickness"] in [ @@ -84,7 +85,7 @@ class FloorRecommendations(Definitions): u_value = get_floor_u_value( floor_type=self.property.floor_type, - area=float(self.property.data["total-floor-area"]), + area=floor_area, perimeter=self.property.perimeter, age_band=self.property.age_band, insulation_thickness=self.property.floor["insulation_thickness"], @@ -92,16 +93,22 @@ class FloorRecommendations(Definitions): ) self.estimated_u_value = u_value + if u_value < self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + return + if self.property.floor["is_suspended"]: # Given the U-value, we recommend underfloor insulation self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts) + return if self.property.floor["is_solid"]: # Given the U-value, we recommend solid floor insulation options which are usually solid foam self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts) + return if self.property.floor["is_to_unheated_space"] or self.property.floor["is_to_external_air"]: self.recommend_floor_insulation(u_value=u_value, parts=self.exposed_floor_insulation_parts) + return raise NotImplementedError("Implement me!") @@ -130,8 +137,9 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + quantity = self.property.floor_area / self.property.number_of_storeys - estimated_cost = cost_per_unit * self.property.floor_area + estimated_cost = cost_per_unit * quantity self.recommendations.append( { @@ -139,7 +147,7 @@ class FloorRecommendations(Definitions): get_recommended_part( part=part, selected_depth=depth, - quantity=self.property.floor_area, + quantity=quantity, quantity_unit=QuantityUnits.m2.value, selected_total_cost=estimated_cost ), diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 74910e07..9205cec7 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -350,17 +350,17 @@ def get_exposed_floor_uvalue(insulation_thickness_str, age_band): unknown_insulation_age_bands = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] # As directed by the documentation, if the insulation thickness is not known, we assume it's # 50mm for these age bands - if insulation_thickness_str in ["below_average", "average", "above_average"] and ( + if insulation_thickness_str in ["below average", "average", "above average"] and ( age_band in unknown_insulation_age_bands ): insulation_thickness = 50 elif insulation_thickness_str in ["none", None]: insulation_thickness = 0 - elif insulation_thickness_str in ["below_average"]: + elif insulation_thickness_str == "below average": insulation_thickness = 50 elif insulation_thickness_str == "average": insulation_thickness = 100 - elif insulation_thickness_str == "above_average": + elif insulation_thickness_str == "above average": insulation_thickness = 150 else: insulation_thickness = int(insulation_thickness_str.replace("mm", "")) diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index f34bbe81..cad5fe24 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -3,6 +3,7 @@ import pytest import os from unittest.mock import Mock from recommendations.FloorRecommendations import FloorRecommendations +from backend.Property import Property # with open( # os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb" @@ -67,7 +68,23 @@ solid_floor_insulation_parts = [ ] -parts = suspended_floor_insulation_parts + solid_floor_insulation_parts +exposed_floor_insulation_parts = [ + { + "type": "exposed_floor_insulation", + "description": "Rockwool Stone Wool insulation", + "depths": [50, 100, 140], + "depth_unit": "mm", + "cost": [8, 11, 15], + "cost_unit": "gbp_sq_meter", + "r_value_per_mm": 0.026315789473684213, + "r_value_unit": "square_meter_kelvin_per_watt", + "thermal_conductivity": 0.038, + "thermal_conductivity_unit": "watt_per_meter_kelvin", + "link": "https://insulation4less.co.uk/products/rockwool-flexi-slab-all-sizes?variant=33409590853685" + }, +] + +parts = suspended_floor_insulation_parts + solid_floor_insulation_parts + exposed_floor_insulation_parts class TestFloorRecommendations: @@ -119,6 +136,7 @@ class TestFloorRecommendations: input_properties[2].perimeter = 20 input_properties[2].wall_type = "solid brick" input_properties[2].floor_type = "suspended" + input_properties[2].number_of_storeys = 1 recommender = FloorRecommendations( property_instance=input_properties[2], @@ -162,6 +180,7 @@ class TestFloorRecommendations: input_properties[4].perimeter = 50 input_properties[4].wall_type = "solid brick" input_properties[4].floor_type = "solid" + input_properties[4].number_of_storeys = 1 recommender = FloorRecommendations( property_instance=input_properties[4], @@ -193,3 +212,124 @@ class TestFloorRecommendations: assert not recommender.property.floor["is_solid"] assert recommender.estimated_u_value is None assert not recommender.recommendations + + def test_exposed_floor_no_insulation(self): + input_property = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property.floor = { + 'original_description': 'To unheated space, no insulation (assumed)', + 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'none' + } + input_property.age_band = "L" + input_property.set_floor_type() + input_property.data = {"floor-level": 0, "property-type": "House"} + input_property.floor_area = 100 + input_property.number_of_storeys = 1 + + recommender = FloorRecommendations( + property_instance=input_property, + materials=exposed_floor_insulation_parts + ) + + assert not recommender.recommendations + + recommender.recommend() + + # Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation + assert not len(recommender.recommendations) + assert recommender.estimated_u_value == 0.22 + + # Now with an older age band + + input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property2.floor = { + 'original_description': 'To unheated space, no insulation (assumed)', + 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'none' + } + input_property2.age_band = "D" + input_property2.set_floor_type() + input_property2.data = {"floor-level": 0, "property-type": "House"} + input_property2.floor_area = 100 + input_property2.number_of_storeys = 1 + + recommender2 = FloorRecommendations( + property_instance=input_property2, + materials=exposed_floor_insulation_parts + ) + + assert not recommender2.recommendations + + recommender2.recommend() + + assert len(recommender2.recommendations) == 1 + + assert recommender2.recommendations[0]["new_u_value"] == 0.23 + assert recommender2.recommendations[0]["starting_u_value"] == 1.2 + assert recommender2.recommendations[0]["cost"] == 1500 + + def test_exposed_floor_below_average_insulated(self): + input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property3.floor = { + 'original_description': 'To unheated space, below average insulation (assumed)', + 'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'below average' + } + input_property3.age_band = "C" + input_property3.set_floor_type() + input_property3.data = {"floor-level": 0, "property-type": "House"} + input_property3.floor_area = 100 + input_property3.number_of_storeys = 1 + + recommender3 = FloorRecommendations( + property_instance=input_property3, + materials=exposed_floor_insulation_parts + ) + + assert not recommender3.recommendations + + recommender3.recommend() + + assert recommender3.estimated_u_value == 0.5 + + assert len(recommender3.recommendations) == 1 + + assert recommender3.recommendations[0]["new_u_value"] == 0.22 + assert recommender3.recommendations[0]["starting_u_value"] == 0.5 + assert recommender3.recommendations[0]["cost"] == 1100 + assert recommender3.recommendations[0]["parts"][0]["depths"] == [100] + + # With average insulation, no recommendations + + input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) + input_property4.floor = { + 'original_description': 'To unheated space, insulated (assumed)', + 'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'average' + } + input_property4.age_band = "C" + input_property4.set_floor_type() + input_property4.data = {"floor-level": 0, "property-type": "House"} + input_property4.floor_area = 100 + input_property4.number_of_storeys = 1 + + recommender4 = FloorRecommendations( + property_instance=input_property4, + materials=exposed_floor_insulation_parts + ) + + assert not recommender4.recommendations + + recommender4.recommend() + + assert recommender4.estimated_u_value is None + + assert len(recommender4.recommendations) == 0 From 22a0bb1ffd9867334fd888059c28be0f9284e66f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Oct 2023 17:32:48 +0800 Subject: [PATCH 33/75] insulation of floor to unheated space complete --- backend/app/plan/router.py | 11 +++++++++++ backend/tests/test_sap_model_prep.py | 25 ++++++++++++------------- recommendations/FloorRecommendations.py | 4 ++-- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d3ea3f83..23ad4262 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -140,6 +140,17 @@ async def trigger_plan(body: PlanTriggerRequest): # with open("materials_by_type", "rb") as f: # materials_by_type = pickle.load(f) + # materials_by_type["floor"].append( + # {'id': 18, 'type': 'exposed_floor_insulation', 'description': 'Rockwool Stone Wool insulation', + # 'depths': [50, 100, 140], 'depth_unit': 'mm', 'cost': [8, 11, 15], + # 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.026315789473684213, + # 'r_value_unit': 'square_meter_kelvin_per_watt', + # 'thermal_conductivity': 0.038, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + # 'link': 'https://insulation4less.co.uk/products/rockwool-flexi-slab-all-sizes?variant=33409590853685', + # 'created_at': datetime(2023, 8, 10, 16, 59, 10, 815531), 'is_active': True} + # + # ) + recommendations = {} recommendations_scoring_data = [] diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index dc793cad..da38cabf 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -10,23 +10,22 @@ from utils.s3 import read_dataframe_from_s3_parquet from tqdm import tqdm -# Handy code for selecting testin data +# Handy code for selecting testing data # import pickle # -# with open("sap_change_dataset.pickle", "rb") as f: +# with open("sap_dataset.pickle", "rb") as f: # sap_change_dataset = pickle.load(f) # # search_from = sap_change_dataset[ -# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) -# ] +# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) & +# sap_change_dataset["is_to_unheated_space"] +# ] # search_from = search_from[ # (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & -# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & +# (search_from["floor_thermal_transmittance_ENDING"] != search_from["floor_thermal_transmittance"]) & # (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & # (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & -# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) & -# (search_from["NUMBER_OPEN_FIREPLACES_STARTING"] > 0) & -# (search_from["NUMBER_OPEN_FIREPLACES_ENDING"] == 0) +# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) # ] # # # Find a record where the only difference is cavity wall getting filled @@ -54,7 +53,7 @@ from tqdm import tqdm # starting_cols.append(starting_col) # # # We want them to be different -# if c == "NUMBER_OPEN_FIREPLACES_ENDING": +# if c == "floor_thermal_transmittance_ENDING": # if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): # same = False # break @@ -82,11 +81,11 @@ from tqdm import tqdm # # compare = pd.concat([start, end], axis=1) # -# ending_lmk = "bab3983fa167717b8bb4a36ef395046d53937f9b880a45bcc751270d72e5de45" -# starting_lmk = "736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef" +# ending_lmk = "1252008839062019090910572351658131" +# starting_lmk = "1252008819542014122308482236142128" # # client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "9 Glebe Road, Asfordby Hill", "postcode": "LE14 3QT"}) +# result = client.domestic.search(params={"address": "Flat 14 Charles House, Freemens Way", "postcode": "CT14 9DL"}) # starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] # ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] @@ -101,7 +100,7 @@ from tqdm import tqdm # ) as f: # cleaning_data = pickle.load(f) -# TODO: Need to do floors, both suspended and solid +# TODO: Need to do floors, suspended and solid and to unheated space class TestSapModelPrep: diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index a78d984e..bc24b6c3 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -58,7 +58,7 @@ class FloorRecommendations(Definitions): ) property_type = self.property.data["property-type"] - floor_area = self.property.floor_area / self.property.number_of_storeys + floor_area = self.property.floor_area / self.property.number_of_floors year_built = self.property.year_built if self.property.floor["another_property_below"] | (self.property.floor["insulation_thickness"] in [ @@ -137,7 +137,7 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - quantity = self.property.floor_area / self.property.number_of_storeys + quantity = self.property.floor_area / self.property.number_of_floors estimated_cost = cost_per_unit * quantity From c96fafa70138ca69be4718537e8cb0a8997a6c4e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Nov 2023 15:57:51 +0900 Subject: [PATCH 34/75] basic implementation of pitched roof area estimate --- backend/Property.py | 9 +++++- recommendations/RoofRecommendations.py | 23 +++++++++++++- recommendations/recommendation_utils.py | 31 +++++++++++++++++++ .../tests/test_roof_recommendations.py | 22 +++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index b0c6b083..1094e7b2 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -11,7 +11,9 @@ from utils.s3 import read_dataframe_from_s3_parquet from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup -from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area +from recommendations.recommendation_utils import ( + estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area +) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') EPC_AUTH_TOKEN = os.environ.get('EPC_AUTH_TOKEN') @@ -79,6 +81,7 @@ class Property(Definitions): self.floor_height = None self.insulation_wall_area = None self.floor_area = None + self.pitched_roof_area = None if epc_client: self.epc_client = epc_client @@ -587,6 +590,10 @@ class Property(Definitions): num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter ) + self.pitched_roof_area = esimtate_pitched_roof_area( + floor_area=self.floor_area / self.number_of_floors, floor_height=self.floor_height + ) + def set_wall_type(self): """ This method sets the wall type of the property, using a simple approach based on the wall description diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index d364dea9..cf7c2a0f 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -54,10 +54,13 @@ class RoofRecommendations: u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) if self.property.roof["is_pitched"]: - # We recommend loft insulation self.recommend_loft_insulation(u_value, insulation_thickness) return + if self.property.roof["is_roof_room"]: + self.recommend_room_roof_insulation(u_value, insulation_thickness) + return + raise NotImplementedError("Implement me") @staticmethod @@ -132,3 +135,21 @@ class RoofRecommendations: ) self.recommendations = recommendations + + def recommend_room_roof_insulation(self, u_value, insulation_thickness): + """ + This method recommends room in roof insulation for properties that have been identified + to possess a room in roof. + + Because we currently have limited data about the construction of the roof, we make the following + assumptions: + 1) The room in roof has a sloped roof. + We will make some basic estimations about the area of the roof given the floor area and the height of the + floors + 2) Insulation of external walls is covered by the wall recommendation class + 3) We assume a "Gable" roof type + + :param u_value: + :param insulation_thickness: + :return: + """ diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 9205cec7..954998f4 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -1,6 +1,7 @@ import math from copy import deepcopy +import numpy as np import pandas as pd from recommendations.rdsap_tables import ( @@ -594,3 +595,33 @@ def convert_thickness_to_numeric(string_thickness): return int(string_thickness.replace("+", "")) return int(string_thickness) + + +def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float: + """ + This function will estimate the area of a pitched roof, given the floor area below the roof and the floor + height of the property. + + Given limited information about the home, this is a very rough method to estimate the roof area and we + assume the the room is a gable roof. + + We assume a roughly average pitch of 45 degrees + + Note that both floor area and height should be in the same units. E.g. if floor area is meters squared, + floor height should be in meters + + :param floor_area: area of the home's floor + :param floor_height: height of the home's floors + :return: Numerical estimate of the surface area of the top of the pitched roof + """ + + # We estimate the length of the wall by just modelling the house as a square + wall_width = np.sqrt(floor_area) + + # We're modelling the roof as two triangles where we know two of the three sides. + # The floor height makes up one side and half of the wall width makes up the other side + slope = np.sqrt(np.square(wall_width / 2) + np.square(floor_height)) + + area = 2 * (slope * wall_width) + + return area diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 8243a35b..b822f829 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -192,3 +192,25 @@ class TestRoofRecommendations: roof_recommender6.recommend() assert len(roof_recommender6.recommendations) == 0 + + def test_uninsulated_room_in_roof(self): + property_instance7 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance7.age_band = "F" + property_instance7.floor_area = 100 + property_instance7.roof = { + 'original_description': 'Roof room(s), no insulation (assumed)', + 'clean_description': 'Roof room(s), no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none' + } + + property_instance7 + + roof_recommender7 = RoofRecommendations( + property_instance=property_instance7, materials=loft_insulation_materials + ) + + assert not roof_recommender7.recommendations + + roof_recommender7.recommend() From 15e48c216596c417640635162e63c7860f6ebf99 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 13 Nov 2023 20:58:00 +0900 Subject: [PATCH 35/75] wrote unit tests for room roof insulation --- recommendations/RoofRecommendations.py | 96 +++++++++++++- .../tests/test_recommendation_utils.py | 56 +++++++++ .../tests/test_roof_recommendations.py | 118 +++++++++++++++++- 3 files changed, 266 insertions(+), 4 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index cf7c2a0f..a33a709b 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -67,6 +67,10 @@ class RoofRecommendations: def make_loft_insulation_description(material, depth): return f"Install {depth}{material['depth_unit']} of {material['description']}" + @staticmethod + def make_room_roof_insulation(material, depth): + return f"Insulate your room roof with {depth}{material['depth_unit']} of {material['description']}" + def recommend_loft_insulation(self, u_value, insulation_thickness): """ @@ -81,6 +85,8 @@ class RoofRecommendations: # from the base layer loft_insulation_materials = [m for m in self.materials if m["type"] == "loft_insulation"] + if not loft_insulation_materials: + raise ValueError("No loft insulation materials found") lowest_selected_u_value = None recommendations = [] @@ -149,7 +155,93 @@ class RoofRecommendations: 2) Insulation of external walls is covered by the wall recommendation class 3) We assume a "Gable" roof type - :param u_value: - :param insulation_thickness: + Further, we recommend internal roof insulation for the room in roof + + The following document contains details around best practices for insulating a room in roof + https://assets.publishing.service.gov.uk/media/61d727d18fa8f50594b59305/retrofit-room-in-roof-insulation-best + -practice.pdf + Of particular interest are the following: + + We also follow advide provided in this article on the Energy Saving Trust website, providing + high level guidance around roof insulation: + https://energysavingtrust.org.uk/advice/roof-and-loft-insulation/ + + To insulate a warm loft, the following advice is given + "An alternative way to insulate your loft is to fit rigid insulation boards between and over the rafters. + Rafters are the sloping timbers that make up the roof itself." + + To then insulate a room roof, the following recommendation is provided: + "If you want to use your loft as a living space, or it is already being used as a living space, + then you need to make sure that all the walls and ceilings between a heated room and an unheated space + are insulated. + + - Sloping ceilings can be insulated in the same way as for a warm roof, + but with a layer of plasterboard on the inside of the insulation. + - Vertical walls can be insulated in the same way. + - Flat ceilings can be insulated like a standard loft. + " + + :param u_value: Current u-value of the roof + :param insulation_thickness: Current insulation thickness of the roof :return: """ + + roof_roof_insulation_materials = [m for m in self.materials if m["type"] == "room_roof_insulation"] + if not roof_roof_insulation_materials: + raise ValueError("No room in roof insulation materials found") + + if self.property.pitched_roof_area is None: + raise ValueError("pitched_roof_area not included as property attribute") + + lowest_selected_u_value = None + recommendations = [] + for material in roof_roof_insulation_materials: + for depth, cost_per_unit in zip(material["depths"], material["cost"]): + # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the + # loft is already partially insulated + if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: + continue + + part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) + + _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) + new_u_value = math.ceil(new_u_value * 100.0) / 100.0 + + # If I have a lowest U value and my new u value is higher than that but lower than the + # diminishing returns threshold, it can be considered + + # If I have a lowest U value and my new u value is lower than the lowest value, it's + # further into the diminishing returns threshold and can shouldn't be + + if is_diminishing_returns( + recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE + ): + continue + + # We allow a small tolerance for error so we don't discount the recommendation entirely + if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + + estimated_cost = cost_per_unit * self.property.pitched_roof_area + + recommendations.append( + { + "parts": [ + get_recommended_part( + part=material, + selected_depth=depth, + quantity=self.property.pitched_roof_area, + quantity_unit=QuantityUnits.m2.value, + selected_total_cost=estimated_cost + ) + ], + "type": "roof_insulation", + "description": self.make_room_roof_insulation(material, depth), + "starting_u_value": u_value, + "new_u_value": new_u_value, + "sap_points": None, + "cost": estimated_cost, + } + ) + + self.recommendations = recommendations diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index dc98d946..b0c8a26b 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -1,3 +1,4 @@ +import numpy as np import pytest import math from unittest.mock import MagicMock @@ -344,3 +345,58 @@ def test_park_home(): assert recommendation_utils.get_floor_u_value( 'suspended', 100, 40, 'A', 'park home', insulation_thickness="20mm" ) == 0 + + +def test_esimtate_pitched_roof_area(): + roof_area1 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=100, floor_height=2 + ) + + assert np.isclose(roof_area1, 107.70329614269008) + + # As the floor height gets bigger, the area should get bigger + roof_area2 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=100, floor_height=3 + ) + + assert np.isclose(roof_area2, 116.61903789690601) + + # As the floor area gets smaller, the area should get smaller + roof_area3 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=100, floor_height=1 + ) + + assert np.isclose(roof_area3, 101.9803902718557) + + # As the floor area decreases, area should decrease + roof_area4 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=50, floor_height=2 + ) + + assert np.isclose(roof_area4, 57.44562646538029) + + # As the floor area increases, area should increase + roof_area5 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=150, floor_height=2 + ) + + assert np.isclose(roof_area5, 157.797338380595) + + zero_roof_area = recommendation_utils.esimtate_pitched_roof_area( + floor_area=0, floor_height=1000 + ) + + assert zero_roof_area == 0 + + # If the floor height zero, we don't have a traingle, it's a flat roof + flat_roof_area = recommendation_utils.esimtate_pitched_roof_area( + floor_area=1000, floor_height=0 + ) + + assert flat_roof_area == 1000 + + zero_roof_area2 = recommendation_utils.esimtate_pitched_roof_area( + floor_area=0, floor_height=0 + ) + + assert zero_roof_area2 == 0 diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index b822f829..3f99535c 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -35,6 +35,19 @@ loft_insulation_materials_150mm_existing = [ } ] +room_roof_insulation_materials = [ + { + 'id': 18, + 'type': 'room_roof_insulation', + 'description': 'Example room roof insulation', + 'depths': [50, 150, 220, 270, 300], 'depth_unit': 'mm', 'cost': [9, 10, 11, 12, 13], + 'cost_unit': 'gbp_sq_meter', + 'r_value_per_mm': 0.022727273, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.044, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': None, 'is_active': True + } +] + class TestRoofRecommendations: @@ -205,12 +218,113 @@ class TestRoofRecommendations: 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none' } - property_instance7 + property_instance7.pitched_roof_area = 110 roof_recommender7 = RoofRecommendations( - property_instance=property_instance7, materials=loft_insulation_materials + property_instance=property_instance7, materials=room_roof_insulation_materials ) assert not roof_recommender7.recommendations roof_recommender7.recommend() + + # Even though we have 3 depths, we only end with 1 due to diminishin returns + assert len(roof_recommender7.recommendations) == 1 + + assert roof_recommender7.recommendations[0]["parts"][0]["depths"] == [270] + + assert roof_recommender7.recommendations[0]["new_u_value"] == 0.14 + assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8 + assert roof_recommender7.recommendations[0]["description"] == \ + "Insulate your room roof with 270mm of Example room roof insulation" + + def test_ceiling_insulated_room_in_roof(self): + property_instance8 = Property(id=8, address1="fake", postcode="fake", epc_client=Mock()) + property_instance8.age_band = "F" + property_instance8.floor_area = 100 + property_instance8.roof = { + 'original_description': 'Roof room(s), ceiling insulated', + 'clean_description': 'Roof room(s), ceiling insulated', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, + 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': 'average' + } + + property_instance8.pitched_roof_area = 110 + + roof_recommender8 = RoofRecommendations( + property_instance=property_instance8, materials=room_roof_insulation_materials + ) + + assert not roof_recommender8.recommendations + + roof_recommender8.recommend() + + # No recommendations in this case + assert not roof_recommender8.recommendations + + def test_insulated_room_in_roof(self): + property_instance9 = Property(id=9, address1="fake", postcode="fake", epc_client=Mock()) + property_instance9.age_band = "F" + property_instance9.floor_area = 100 + property_instance9.roof = { + 'original_description': 'Roof room(s), insulated (assumed)', + 'clean_description': 'Roof room(s), insulated', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average' + } + + property_instance9.pitched_roof_area = 110 + + roof_recommender9 = RoofRecommendations( + property_instance=property_instance9, materials=room_roof_insulation_materials + ) + + assert not roof_recommender9.recommendations + + roof_recommender9.recommend() + + # No recommendations in this case + assert not roof_recommender9.recommendations + + def test_limited_insulated_room_in_roof(self): + property_instance10 = Property(id=10, address1="fake", postcode="fake", epc_client=Mock()) + property_instance10.age_band = "F" + property_instance10.floor_area = 100 + property_instance10.roof = { + 'original_description': 'Roof room(s), limited insulation (assumed)', + 'clean_description': 'Roof room(s), limited insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': 'below average' + } + + property_instance10.pitched_roof_area = 110 + + roof_recommender10 = RoofRecommendations( + property_instance=property_instance10, materials=room_roof_insulation_materials + ) + + assert not roof_recommender10.recommendations + + roof_recommender10.recommend() + + assert len(roof_recommender10.recommendations) == 2 + + assert roof_recommender10.recommendations[0]["parts"][0]["depths"] == [220] + assert roof_recommender10.recommendations[1]["parts"][0]["depths"] == [270] + + assert roof_recommender10.recommendations[0]["new_u_value"] == 0.16 + assert roof_recommender10.recommendations[1]["new_u_value"] == 0.14 + + assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.8 + assert roof_recommender10.recommendations[1]["starting_u_value"] == 0.8 + + assert roof_recommender10.recommendations[0]["description"] == \ + "Insulate your room roof with 220mm of Example room roof insulation" + assert roof_recommender10.recommendations[1]["description"] == \ + "Insulate your room roof with 270mm of Example room roof insulation" From 7ae49d35e4feee804a95209b930a33b8f9983e7b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 10:43:14 +0000 Subject: [PATCH 36/75] Adding flat roof insulation --- .idea/misc.xml | 3 + recommendations/RoofRecommendations.py | 55 +++++++++++++++---- .../tests/test_roof_recommendations.py | 25 +++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 3b05c6ac..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,5 +1,8 @@ + + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index da38cabf..d1a643ee 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -195,7 +195,7 @@ class TestSapModelPrep: 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.7, - 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'none', + 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average', 'external_insulation_ENDING': False, 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.64, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': True, 'is_solid': False, 'another_property_below': False, @@ -253,7 +253,29 @@ class TestSapModelPrep: 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', 'fuel_type_ENDING': 'oil', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 44.77882152472145, 'estimated_perimeter_ENDING': 44.77882152472145 + 'estimated_perimeter_STARTING': 44.77882152472145, 'estimated_perimeter_ENDING': 44.77882152472145, + 'HOT_WATER_ENERGY_EFF_STARTING': "Good", + "FLOOR_ENERGY_EFF_STARTING": "Unknown", + "WINDOWS_ENERGY_EFF_STARTING": "Good", + "WALLS_ENERGY_EFF_STARTING": "Poor", + "SHEATING_ENERGY_EFF_STARTING": "Unknown", + "ROOF_ENERGY_EFF_STARTING": "Very Poor", + "MAINHEAT_ENERGY_EFF_STARTING": "Average", + "MAINHEATC_ENERGY_EFF_STARTING": "Good", + "LIGHTING_ENERGY_EFF_STARTING": "Average", + "POTENTIAL_ENERGY_EFFICIENCY": 64, + "ENVIRONMENT_IMPACT_POTENTIAL": 53, + "ENERGY_CONSUMPTION_POTENTIAL": 177.0, + "CO2_EMISSIONS_POTENTIAL": 5.7, + "HOT_WATER_ENERGY_EFF_ENDING": "Good", + "FLOOR_ENERGY_EFF_ENDING": "Unknown", + "WINDOWS_ENERGY_EFF_ENDING": "Good", + "WALLS_ENERGY_EFF_ENDING": "Good", + "SHEATING_ENERGY_EFF_ENDING": "Unknown", + "ROOF_ENERGY_EFF_ENDING": "Very Poor", + "MAINHEAT_ENERGY_EFF_ENDING": "Average", + "MAINHEATC_ENERGY_EFF_ENDING": "Good", + "LIGHTING_ENERGY_EFF_ENDING": "Average", } home = Property( @@ -321,10 +343,9 @@ class TestSapModelPrep: continue if c == "walls_insulation_thickness_ENDING": - print("Add back in the checks") - continue assert row[c] == "average" - assert test_record[c] == "above average" + assert test_record[c].values[0] == "above average" + continue assert test_record[c].values[0] == row[c] From 4d51410ff30fbef541bc037b684c05eca4ee4589 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 11:51:39 +0000 Subject: [PATCH 40/75] Fixing sap model process setup --- backend/tests/test_sap_model_prep.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index d1a643ee..52104eab 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -473,7 +473,29 @@ class TestSapModelPrep: 'fuel_type_ENDING': 'electricity', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', 'estimated_perimeter_STARTING': 35.4964786985977, - 'estimated_perimeter_ENDING': 35.4964786985977 + 'estimated_perimeter_ENDING': 35.4964786985977, + 'HOT_WATER_ENERGY_EFF_STARTING': "Very Poor", + "FLOOR_ENERGY_EFF_STARTING": "Unknown", + "WINDOWS_ENERGY_EFF_STARTING": "Average", + "WALLS_ENERGY_EFF_STARTING": "Very Poor", + "SHEATING_ENERGY_EFF_STARTING": "Unknown", + "ROOF_ENERGY_EFF_STARTING": "Unknown", + "MAINHEAT_ENERGY_EFF_STARTING": "Very Poor", + "MAINHEATC_ENERGY_EFF_STARTING": "Good", + "LIGHTING_ENERGY_EFF_STARTING": "Poor", + "POTENTIAL_ENERGY_EFFICIENCY": 71, + "ENVIRONMENT_IMPACT_POTENTIAL": 51, + "ENERGY_CONSUMPTION_POTENTIAL": 307, + "CO2_EMISSIONS_POTENTIAL": 3.6, + 'HOT_WATER_ENERGY_EFF_ENDING': "Very Poor", + "FLOOR_ENERGY_EFF_ENDING": "Unknown", + "WINDOWS_ENERGY_EFF_ENDING": "Average", + "WALLS_ENERGY_EFF_ENDING": "Good", + "SHEATING_ENERGY_EFF_ENDING": "Unknown", + "ROOF_ENERGY_EFF_ENDING": "Unknown", + "MAINHEAT_ENERGY_EFF_ENDING": "Very Poor", + "MAINHEATC_ENERGY_EFF_ENDING": "Good", + "LIGHTING_ENERGY_EFF_ENDING": "Poor", } home2 = Property( From 314ad8719df424a182d514d69953fa428aa1deba Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 12:12:43 +0000 Subject: [PATCH 41/75] fixing sap model prep unit tests --- backend/tests/test_sap_model_prep.py | 48 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 52104eab..ab7cd678 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -692,7 +692,29 @@ class TestSapModelPrep: 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 41.634120622393354, 'estimated_perimeter_ENDING': 41.634120622393354 + 'estimated_perimeter_STARTING': 41.634120622393354, 'estimated_perimeter_ENDING': 41.634120622393354, + 'HOT_WATER_ENERGY_EFF_STARTING': "Good", + "FLOOR_ENERGY_EFF_STARTING": "Unknown", + "WINDOWS_ENERGY_EFF_STARTING": "Average", + "WALLS_ENERGY_EFF_STARTING": "Very Poor", + "SHEATING_ENERGY_EFF_STARTING": "Unknown", + "ROOF_ENERGY_EFF_STARTING": "Very Poor", + "MAINHEAT_ENERGY_EFF_STARTING": "Good", + "MAINHEATC_ENERGY_EFF_STARTING": "Average", + "LIGHTING_ENERGY_EFF_STARTING": "Average", + "POTENTIAL_ENERGY_EFFICIENCY": 80, + "ENVIRONMENT_IMPACT_POTENTIAL": 75, + "ENERGY_CONSUMPTION_POTENTIAL": 152, + "CO2_EMISSIONS_POTENTIAL": 2.9, + 'HOT_WATER_ENERGY_EFF_ENDING': "Good", + "FLOOR_ENERGY_EFF_ENDING": "Unknown", + "WINDOWS_ENERGY_EFF_ENDING": "Average", + "WALLS_ENERGY_EFF_ENDING": "Very Poor", + "SHEATING_ENERGY_EFF_ENDING": "Unknown", + "ROOF_ENERGY_EFF_ENDING": "Very Poor", + "MAINHEAT_ENERGY_EFF_ENDING": "Good", + "MAINHEATC_ENERGY_EFF_ENDING": "Average", + "LIGHTING_ENERGY_EFF_ENDING": "Average", } home3 = Property( @@ -878,7 +900,29 @@ class TestSapModelPrep: 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 37.54197650630557, 'estimated_perimeter_ENDING': 37.54197650630557 + 'estimated_perimeter_STARTING': 37.54197650630557, 'estimated_perimeter_ENDING': 37.54197650630557, + 'HOT_WATER_ENERGY_EFF_STARTING': "Good", + "FLOOR_ENERGY_EFF_STARTING": "Unknown", + "WINDOWS_ENERGY_EFF_STARTING": "Average", + "WALLS_ENERGY_EFF_STARTING": "Very Poor", + "SHEATING_ENERGY_EFF_STARTING": "Unknown", + "ROOF_ENERGY_EFF_STARTING": "Good", + "MAINHEAT_ENERGY_EFF_STARTING": "Good", + "MAINHEATC_ENERGY_EFF_STARTING": "Average", + "LIGHTING_ENERGY_EFF_STARTING": "Average", + "POTENTIAL_ENERGY_EFFICIENCY": 78, + "ENVIRONMENT_IMPACT_POTENTIAL": 76, + "ENERGY_CONSUMPTION_POTENTIAL": 153, + "CO2_EMISSIONS_POTENTIAL": 2.4, + 'HOT_WATER_ENERGY_EFF_ENDING': "Good", + "FLOOR_ENERGY_EFF_ENDING": "Unknown", + "WINDOWS_ENERGY_EFF_ENDING": "Average", + "WALLS_ENERGY_EFF_ENDING": "Very Poor", + "SHEATING_ENERGY_EFF_ENDING": "Unknown", + "ROOF_ENERGY_EFF_ENDING": "Good", + "MAINHEAT_ENERGY_EFF_ENDING": "Good", + "MAINHEATC_ENERGY_EFF_ENDING": "Average", + "LIGHTING_ENERGY_EFF_ENDING": "Average", } home4 = Property( From ccc0d8603b81fa8a1d9bc29a5872f775d7322a40 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 12:16:11 +0000 Subject: [PATCH 42/75] fixed property unit tests --- backend/tests/test_property.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index d7028bc6..b376db9e 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -227,7 +227,8 @@ class TestProperty: "mainheat-description": [{"original_description": "Main Heating Description"}], "hotwater-description": [{"original_description": "Hot Water Description"}], "lighting-description": [{"original_description": "Good Lighting Efficiency"}], - "floor-description": [{"original_description": "Floor Description", "is_suspended": True}] + "floor-description": [ + {"original_description": "Floor Description", "is_suspended": True, "another_property_below": False}] } return mock_cleaner @@ -317,7 +318,9 @@ class TestProperty: } property_instance.floor = { - "is_suspended": False + "is_suspended": False, + "another_property_below": False, + "is_solid": True } # Assert backup cleaning has been applied From 763b88cd1edc46c7201cf5ea334dfef8c4b256c0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 13:55:46 +0000 Subject: [PATCH 43/75] fix tests for convert thickness to numeric --- .../tests/test_recommendation_utils.py | 19 ++++++++++++------- .../tests/test_roof_recommendations.py | 9 ++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index b0c8a26b..22280ed5 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -280,14 +280,19 @@ class TestRecommendationUtils: def test_convert_thickness_to_numeric(self): - assert recommendation_utils.convert_thickness_to_numeric("none") == 0 - assert recommendation_utils.convert_thickness_to_numeric("below average") == 50 - assert recommendation_utils.convert_thickness_to_numeric("average") == 100 - assert recommendation_utils.convert_thickness_to_numeric("above average") == 270 + assert recommendation_utils.convert_thickness_to_numeric("none", True) == 0 + assert recommendation_utils.convert_thickness_to_numeric("below average", True) == 50 + assert recommendation_utils.convert_thickness_to_numeric("average", True) == 100 + assert recommendation_utils.convert_thickness_to_numeric("above average", True) == 270 - assert recommendation_utils.convert_thickness_to_numeric("300+") == 300 - assert recommendation_utils.convert_thickness_to_numeric("400+") == 400 - assert recommendation_utils.convert_thickness_to_numeric("270") == 270 + assert recommendation_utils.convert_thickness_to_numeric("300+", True) == 300 + assert recommendation_utils.convert_thickness_to_numeric("400+", True) == 400 + assert recommendation_utils.convert_thickness_to_numeric("270", True) == 270 + + assert recommendation_utils.convert_thickness_to_numeric("none", False) == 0 + assert recommendation_utils.convert_thickness_to_numeric("below average", False) == 100 + assert recommendation_utils.convert_thickness_to_numeric("average", False) == 270 + assert recommendation_utils.convert_thickness_to_numeric("above average", False) == 270 def test_estimate_perimeter_regular_inputs(): diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index a3cc9266..37cc2daf 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -362,21 +362,16 @@ class TestRoofRecommendations: roof_recommender11.recommend() - assert len(roof_recommender11.recommendations) == 2 + assert len(roof_recommender11.recommendations) == 1 assert roof_recommender11.recommendations[0]["parts"][0]["depths"] == [270] - assert roof_recommender11.recommendations[1]["parts"][0]["depths"] == [300] - assert roof_recommender11.recommendations[0]["new_u_value"] == 0.16 - assert roof_recommender11.recommendations[1]["new_u_value"] == 0.14 + assert roof_recommender11.recommendations[0]["new_u_value"] == 0.11 assert roof_recommender11.recommendations[0]["starting_u_value"] == 2.3 - assert roof_recommender11.recommendations[1]["starting_u_value"] == 2.3 assert roof_recommender11.recommendations[0]["description"] == \ "Insulate the home's flat roof with 270mm of Example flat roof insulation" - assert roof_recommender11.recommendations[1]["description"] == \ - "Insulate the home's flat roof with 300mm of Example flat roof insulation" def test_flat_insulated(self): property_instance12 = Property(id=12, address1="fake", postcode="fake", epc_client=Mock()) From ee0fd1452bd0bd8a53b65ac9f7ca0d9482944b36 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 14:03:42 +0000 Subject: [PATCH 44/75] fixed unit tests --- .../tests/test_floor_recommendations.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index cad5fe24..82ba7cf4 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -115,6 +115,8 @@ class TestFloorRecommendations: assert obj.property def test_other_premises_below(self, input_properties): + input_properties[0].floor_area = 100 + input_properties[0].number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_properties[0], materials=parts @@ -136,7 +138,7 @@ class TestFloorRecommendations: input_properties[2].perimeter = 20 input_properties[2].wall_type = "solid brick" input_properties[2].floor_type = "suspended" - input_properties[2].number_of_storeys = 1 + input_properties[2].number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_properties[2], @@ -145,7 +147,7 @@ class TestFloorRecommendations: assert recommender.estimated_u_value is None recommender.recommend() assert recommender.property.floor["is_suspended"] - assert recommender.estimated_u_value == 0.39 + assert recommender.estimated_u_value == 0.66 assert recommender.recommendations types = {part["type"] for x in recommender.recommendations for part in x["parts"]} @@ -158,6 +160,8 @@ class TestFloorRecommendations: does not need floor insulation :return: """ + input_properties[3].floor_area = 100 + input_properties[3].number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_properties[3], materials=parts @@ -180,7 +184,7 @@ class TestFloorRecommendations: input_properties[4].perimeter = 50 input_properties[4].wall_type = "solid brick" input_properties[4].floor_type = "solid" - input_properties[4].number_of_storeys = 1 + input_properties[4].number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_properties[4], @@ -190,7 +194,7 @@ class TestFloorRecommendations: recommender.recommend() assert not recommender.property.floor["is_suspended"] assert recommender.property.floor["is_solid"] - assert recommender.estimated_u_value == 0.71 + assert recommender.estimated_u_value == 0.73 assert recommender.recommendations types = {part["type"] for x in recommender.recommendations for part in x["parts"]} @@ -202,6 +206,8 @@ class TestFloorRecommendations: This is another description we see when there is a property below """ + input_properties[6].floor_area = 100 + input_properties[6].number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_properties[6], materials=parts @@ -226,7 +232,7 @@ class TestFloorRecommendations: input_property.set_floor_type() input_property.data = {"floor-level": 0, "property-type": "House"} input_property.floor_area = 100 - input_property.number_of_storeys = 1 + input_property.number_of_floors = 1 recommender = FloorRecommendations( property_instance=input_property, @@ -255,7 +261,7 @@ class TestFloorRecommendations: input_property2.set_floor_type() input_property2.data = {"floor-level": 0, "property-type": "House"} input_property2.floor_area = 100 - input_property2.number_of_storeys = 1 + input_property2.number_of_floors = 1 recommender2 = FloorRecommendations( property_instance=input_property2, @@ -285,7 +291,7 @@ class TestFloorRecommendations: input_property3.set_floor_type() input_property3.data = {"floor-level": 0, "property-type": "House"} input_property3.floor_area = 100 - input_property3.number_of_storeys = 1 + input_property3.number_of_floors = 1 recommender3 = FloorRecommendations( property_instance=input_property3, @@ -319,7 +325,7 @@ class TestFloorRecommendations: input_property4.set_floor_type() input_property4.data = {"floor-level": 0, "property-type": "House"} input_property4.floor_area = 100 - input_property4.number_of_storeys = 1 + input_property4.number_of_floors = 1 recommender4 = FloorRecommendations( property_instance=input_property4, From 1846b8dc1459d99b32e2673dea7b1a4b4f2e7882 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 14:09:22 +0000 Subject: [PATCH 45/75] remove get_cleaned from unit tests --- backend/tests/test_sap_model_prep.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index ab7cd678..d512a499 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -5,8 +5,9 @@ from etl.epc.settings import COLUMNS_TO_MERGE_ON from epc_api.client import EpcClient import pandas as pd import pytest +import msgpack -from utils.s3 import read_dataframe_from_s3_parquet +from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 from tqdm import tqdm @@ -113,6 +114,12 @@ class TestSapModelPrep: @pytest.fixture def cleaned(self): + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + cleaned = msgpack.unpackb(cleaned, raw=False) return get_cleaned() def test_fill_cavity_wall(self, cleaned, cleaning_data): From dfa5e798e1db862902f67d9995963861d7443138 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 14:12:55 +0000 Subject: [PATCH 46/75] added aws credentials to unit test github workflow --- .github/workflows/unit_tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1fb79ce4..39d285f2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -26,6 +26,12 @@ jobs: run: | python -m pip install --upgrade pip pip install -r model_data/requirements/requirements.txt + - name: Set dev AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 - name: Run tests with pytest run: | pip install -r model_data/requirements/dev.txt From 262517be46460dfe330f65604638f669440ae20c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 14:15:39 +0000 Subject: [PATCH 47/75] fixing stupid bug --- backend/tests/test_sap_model_prep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index d512a499..887e8e6e 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -120,7 +120,7 @@ class TestSapModelPrep: ) cleaned = msgpack.unpackb(cleaned, raw=False) - return get_cleaned() + return cleaned def test_fill_cavity_wall(self, cleaned, cleaning_data): """ From d703447c6a9ee2b600ce5c1949f864903adde016 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 15:39:51 +0000 Subject: [PATCH 48/75] Adding new materials to Materials enum --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 3 + backend/app/plan/temp_script_for_flight.py | 176 --------------------- recommendations/RoofRecommendations.py | 17 +- 5 files changed, 12 insertions(+), 188 deletions(-) delete mode 100644 backend/app/plan/temp_script_for_flight.py diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 1dc47276..812c1ebb 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -15,6 +15,9 @@ class MaterialType(enum.Enum): cavity_wall_insulation = "cavity_wall_insulation" mechanical_ventilation = "mechanical_ventilation" loft_insulation = "loft_insulation" + exposed_floor_insulation = "exposed_floor_insulation" + flat_roof_insulation = "flat_roof_insulation" + room_roof_insulation = "room_roof_insulation" class DepthUnit(enum.Enum): diff --git a/backend/app/plan/temp_script_for_flight.py b/backend/app/plan/temp_script_for_flight.py deleted file mode 100644 index 9170b4c1..00000000 --- a/backend/app/plan/temp_script_for_flight.py +++ /dev/null @@ -1,176 +0,0 @@ -from datetime import datetime - -import pandas as pd -from epc_api.client import EpcClient -from fastapi import APIRouter, Depends -from sqlalchemy.exc import IntegrityError, OperationalError -from sqlalchemy.orm import sessionmaker -from starlette.responses import Response - -from backend.app.config import get_settings -from backend.app.db.connection import db_engine -from backend.app.db.functions.materials_functions import get_materials -from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations -from backend.app.db.functions.property_functions import ( - create_property, create_property_details_epc, create_property_targets, update_property_data -) -from backend.app.db.functions.recommendations_functions import ( - create_plan, create_plan_recommendations, upload_recommendations -) -from backend.app.db.models.portfolio import rating_lookup -from backend.app.dependencies import validate_token -from backend.app.plan.schemas import PlanTriggerRequest -from backend.app.plan.utils import ( - create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id -) -from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 - -from backend.ml_models.sap_change_model.api import SAPChangeModelAPI -from backend.Property import Property -from etl.epc.DataProcessor import DataProcessor -from etl.epc.settings import COLUMNS_TO_MERGE_ON -from recommendations.FloorRecommendations import FloorRecommendations -from recommendations.optimiser.CostOptimiser import CostOptimiser -from recommendations.optimiser.GainOptimiser import GainOptimiser -from recommendations.optimiser.optimiser_functions import prepare_input_measures -from recommendations.WallRecommendations import WallRecommendations -from utils.logger import setup_logger -from utils.s3 import read_dataframe_from_s3_parquet - -logger = setup_logger() - -import pickle - -with open('local_data.pickle', 'rb') as f: - local_data = pickle.load(f) - -with open("property_dimensions.pickle", "rb") as f: - property_dimensions = pickle.load(f) - -with open("sap_change_dataset.pickle", "rb") as f: - sap_change_dataset = pickle.load(f) - -created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - -plan_input = local_data["plan_input"] -uprn_filenames = local_data["uprn_filenames"] -local_property_data = local_data["local_property_data"] -materials = local_data["materials"] -materials_by_type = filter_materials(materials) -cleaned = local_data["cleaned"] -cleaning_data = local_data["cleaning_data"] - -# Need to find some proper materials -materials_by_type["walls"] += [ - {'id': 4, 'type': 'cavity_wall_insulation', 'description': 'Example Material 1', - 'depths': None, - 'depth_unit': None, 'cost': 20, - 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.0278, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.036, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': None, 'created_at': None, 'is_active': True}, - {'id': 10, 'type': "cavity_wall_insulation", 'description': 'Example Material 2', - 'depths': None, 'depth_unit': None, 'cost': 25, 'cost_unit': 'gbp_sq_meter', - 'r_value_per_mm': 0.02631579, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': None, - 'created_at': None, 'is_active': True} -] - -epc_client = EpcClient(auth_token="NO-TOKEN") - -input_properties = [] -for i, config in enumerate(plan_input): - property_id = local_property_data[i]["id"] - input_properties.append( - Property( - postcode=config['postcode'], - address1=config['address'], - epc_client=epc_client, - id=property_id - ) - ) - -logger.info("Getting EPC, and spatial data") -for i, p in enumerate(input_properties): - p.data = local_property_data[i]["data"] - p.uprn = local_property_data[i]["uprn"] - p.id = local_property_data[i]["id"] - p.full_sap_epc = local_property_data[i]["full_sap_epc"] - p.old_data = local_property_data[i]["old_data"] - p.is_listed = False - p.in_conservation_area = False - p.is_heritage = False - - p.set_year_built() - - # TODO: TESTING - p.data['number-habitable-rooms'] = 3 - -recommendations = {} -recommendations_scoring_data = [] - -for p in input_properties: - property_recommendations = [] - - # Property recommendations - p.get_components(cleaned) - - # Floor recommendations - floor_recommender = FloorRecommendations( - property_instance=p, - materials=materials_by_type["floor"], - ) - floor_recommender.recommend() - - if floor_recommender.recommendations: - property_recommendations.append(floor_recommender.recommendations) - - # Wall recommendations - - wall_recomender = WallRecommendations( - property_instance=p, - materials=materials_by_type["walls"] - ) - wall_recomender.recommend() - - if wall_recomender.recommendations: - property_recommendations.append(wall_recomender.recommendations) - - # We insert temporary ids into the recommendations which is important for the optimiser later - property_recommendations = insert_temp_recommendation_id(property_recommendations) - - if not property_recommendations: - continue - - recommendations[p.id] = property_recommendations - - # Finally, we'll prepare data for predicting the impact on SAP - # TODO: We should use the cleaned data from get_components in the data rather than the raw - # values. We should create a method in Property which takes the EPC data and inserts the cleaned - # data - - data_processor = DataProcessor(None, newdata=True) - data_processor.insert_data(pd.DataFrame([p.data.copy()])) - data_processor.pre_process() - - starting_epc_data = data_processor.get_component_features(suffix="_STARTING") - ending_epc_data = data_processor.get_component_features(suffix="_ENDING") - fixed_data = data_processor.get_fixed_features() - - # We update the ending record with the recommended updates and we set lodgement date to today - ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at - - for recommendations_by_type in property_recommendations: - for rec in recommendations_by_type: - scoring_dict = create_recommendation_scoring_data( - property=p, - recommendation=rec, - starting_epc_data=starting_epc_data, - ending_epc_data=ending_epc_data, - fixed_data=fixed_data, - ) - - recommendations_scoring_data.append(scoring_dict) - -# cleanup -del data_processor diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 283370ac..e5200904 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -47,7 +47,8 @@ class RoofRecommendations: # Building regulations part L recommend installing at least 270mm of insulation, however generally we # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation - if insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM: + # This only holds true for pitched roofs + if (insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]: return # If we have a u-value already, need to implement this @@ -61,7 +62,7 @@ class RoofRecommendations: return if self.property.roof["is_roof_room"]: - self.recommend_room_roof_insulation(u_value, insulation_thickness) + self.recommend_room_roof_insulation(u_value) return raise NotImplementedError("Implement me") @@ -125,8 +126,9 @@ class RoofRecommendations: for depth, cost_per_unit in zip(material["depths"], material["cost"]): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the - # loft is already partially insulated - if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: + # loft is already partially insulated. + # Note: This requirement is only for loft insulation + if ((depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: continue part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) @@ -178,7 +180,7 @@ class RoofRecommendations: self.recommendations = recommendations - def recommend_room_roof_insulation(self, u_value, insulation_thickness): + def recommend_room_roof_insulation(self, u_value): """ This method recommends room in roof insulation for properties that have been identified to possess a room in roof. @@ -217,7 +219,6 @@ class RoofRecommendations: - Flat ceilings can be insulated like a standard loft. :param u_value: Current u-value of the roof - :param insulation_thickness: Current insulation thickness of the roof :return: """ @@ -232,10 +233,6 @@ class RoofRecommendations: recommendations = [] for material in roof_roof_insulation_materials: for depth, cost_per_unit in zip(material["depths"], material["cost"]): - # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the - # loft is already partially insulated - if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: - continue part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) From eed5a0baa358fd55c4b063e394ada697ce9a9dd6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 16:00:00 +0000 Subject: [PATCH 49/75] added property get_components template, when there is no component --- backend/Property.py | 8 ++++- backend/app/plan/router.py | 30 ------------------- backend/app/plan/utils.py | 2 +- recommendations/RoofRecommendations.py | 10 +++++++ recommendations/recommendation_utils.py | 3 ++ .../tests/test_roof_recommendations.py | 23 ++++++++++++++ 6 files changed, 44 insertions(+), 32 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 1094e7b2..2359ea6a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -288,10 +288,16 @@ class Property(Definitions): for description, attribute in cleaned.items(): if self.data[description] in self.DATA_ANOMALY_MATCHES: + template = cleaned[description][0] + fill_dict = dict(zip(template.keys(), [None] * len(template))) + fill_dict.update({ + "original_description": self.data[description], + "clean_description": self.data[description], + }) setattr( self, self.ATTRIBUTE_MAP[description], - {"original_description": self.data[description], "clean_description": self.data[description]} + fill_dict, ) continue diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 23ad4262..e531896e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -39,7 +39,6 @@ from recommendations.optimiser.optimiser_functions import prepare_input_measures from recommendations.WallRecommendations import WallRecommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet -from tqdm import tqdm logger = setup_logger() @@ -122,35 +121,6 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - # import pickle - # with open("input_properties.pickle", "rb") as f: - # input_properties = pickle.load(f) - # - # import pickle - # with open("new_sap_dataset.pickle", "rb") as f: - # new_sap_dataset = pickle.load(f) - - # import pickle - # with open("cleaned.pickle", "rb") as f: - # cleaned = pickle.load(f) - - # with open("sap_dataset.pickle", "rb") as f: - # sap_dataset = pickle.load(f) - - # with open("materials_by_type", "rb") as f: - # materials_by_type = pickle.load(f) - - # materials_by_type["floor"].append( - # {'id': 18, 'type': 'exposed_floor_insulation', 'description': 'Rockwool Stone Wool insulation', - # 'depths': [50, 100, 140], 'depth_unit': 'mm', 'cost': [8, 11, 15], - # 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.026315789473684213, - # 'r_value_unit': 'square_meter_kelvin_per_watt', - # 'thermal_conductivity': 0.038, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - # 'link': 'https://insulation4less.co.uk/products/rockwool-flexi-slab-all-sizes?variant=33409590853685', - # 'created_at': datetime(2023, 8, 10, 16, 59, 10, 815531), 'is_active': True} - # - # ) - recommendations = {} recommendations_scoring_data = [] diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 71a61be1..36e90d61 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -17,7 +17,7 @@ def filter_materials(materials): "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"], "ventilation": ["mechanical_ventilation"], - "roof": ["loft_insulation"] + "roof": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"] } materials = [row2dict(material) for material in materials] diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index e5200904..bfa63908 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -36,6 +36,10 @@ class RoofRecommendations: self.materials = materials def recommend(self): + + if self.property.roof["has_dwelling_above"]: + return + u_value = self.property.roof["thermal_transmittance"] insulation_thickness = convert_thickness_to_numeric( @@ -53,6 +57,12 @@ class RoofRecommendations: # If we have a u-value already, need to implement this if u_value: + if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + # The floor is already compliant + return + + if self.property.data["transaction-type"] == "new dwelling": + return raise NotImplementedError("Implement me") u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 13f58fd9..063a274c 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -585,6 +585,9 @@ def convert_thickness_to_numeric(string_thickness, is_pitched): :return: integer measure of insulation thickness """ + if string_thickness is None: + return 0 + if is_pitched: lookup = { "none": 0, diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 37cc2daf..551407da 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -427,3 +427,26 @@ class TestRoofRecommendations: assert roof_recommender13.recommendations[0]["description"] == \ "Insulate the home's flat roof with 220mm of Example flat roof insulation" + + def test_property_above(self): + property_instance14 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance14.age_band = "F" + property_instance14.floor_area = 100 + property_instance14.roof = { + 'original_description': '(other premises above)', + 'clean_description': '(other premises above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False, + 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, + 'insulation_thickness': None + } + + roof_recommender14 = RoofRecommendations( + property_instance=property_instance14, materials=loft_insulation_materials + ) + + assert not roof_recommender14.recommendations + + roof_recommender14.recommend() + + assert not roof_recommender14.recommendations From beb45d451d950f38e20900a7701d76ab8fabc38d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 16:52:47 +0000 Subject: [PATCH 50/75] fetch old data construction age band if missing --- backend/Property.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 2359ea6a..1cebc4a9 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1,4 +1,5 @@ from datetime import datetime +from collections import Counter import re import os import pandas as pd @@ -334,8 +335,24 @@ class Property(Definitions): raise ValueError("Property does not contain data") self.construction_age_band = DataProcessor.clean_construction_age_band(self.data["construction-age-band"]) + if self.construction_age_band in self.DATA_ANOMALY_MATCHES: + if self.old_data: + # Take the most recent + max_datetime = max( + [x["lodgement-datetime"] for x in self.old_data if + x["construction-age-band"] not in self.DATA_ANOMALY_MATCHES] + ) + most_recent = [x for x in self.old_data if x["lodgement-datetime"] == max_datetime] + + self.construction_age_band = DataProcessor.clean_construction_age_band( + most_recent[0]["construction-age-band"] + ) + self.age_band = england_wales_age_band_lookup.get(self.construction_age_band) + if self.age_band is None: + raise ValueError("age_band is missing") + def set_spatial(self, spatial: pd.DataFrame): """ Sets whether the property is in a conservation area given the output of the ConservationAreaClient From be9a9601862165cf0582237dba0976fb8292f0e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 17:03:14 +0000 Subject: [PATCH 51/75] Added default age band for new builds --- backend/Property.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 1cebc4a9..259ca724 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -350,6 +350,10 @@ class Property(Definitions): self.age_band = england_wales_age_band_lookup.get(self.construction_age_band) + if (self.data["transaction-type"] == "new dwelling") and (self.age_band is None): + self.age_band = "L" + self.construction_age_band = 'England and Wales: 2012 onwards' + if self.age_band is None: raise ValueError("age_band is missing") From 76a03cdf23189146e70ed27727f4960b5a0f08ef Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 11:12:34 +0000 Subject: [PATCH 52/75] Final fixes to get portfolio working --- backend/Property.py | 1 + backend/app/plan/router.py | 13 ++++++++++--- etl/epc/DataProcessor.py | 12 +++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 259ca724..370eca06 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -749,6 +749,7 @@ class Property(Definitions): "TOTAL_FLOOR_AREA": self.floor_area, **epc_raw_data, "BUILT_FORM": built_form, + "POSTCODE": self.data["postcode"], } return property_data diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e531896e..4064452f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -211,11 +211,18 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Preparing data for scoring in sap change api") recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) - # Perform the same cleaning as in the model + # Perform the same cleaning as in the model - first clean number of room variables though recommendations_scoring_data = DataProcessor.apply_averages_cleaning( data_to_clean=recommendations_scoring_data, cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'], + colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"], + ) + + recommendations_scoring_data = DataProcessor.apply_averages_cleaning( + data_to_clean=recommendations_scoring_data, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"], ).drop(columns=["LOCAL_AUTHORITY"]) recommendations_scoring_data = DataProcessor.clean_missings_after_description_process( @@ -303,7 +310,7 @@ async def trigger_plan(body: PlanTriggerRequest): # 3) the recommendations logger.info("Uploading recommendations to the database") - for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): + for i in range(0, len(input_properties), BATCH_SIZE): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 3ef485b8..0587fdbe 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -179,7 +179,6 @@ class DataProcessor: # We have some non-standard construction age bands which we'll clean for matching if not self.newdata: self.standardise_construction_age_band() - self.clean_missing_rooms() self.recast_df_columns( @@ -451,7 +450,7 @@ class DataProcessor: self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0) @staticmethod - def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on): + def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on, colnames=None): """ Clean the input DataFrame using averages from a cleaning DataFrame. @@ -459,11 +458,16 @@ class DataProcessor: :param cleaning_data: DataFrame containing data for cleaning. :param cols_to_merge_on: Columns on which merging is based. We pass cols_to_merge_on to this function as this differs depending on where the function is being used. + :param colnames: If specified can be used to state exactly which columns to clean :return: Cleaned DataFrame. """ + # The desired colnames to clean - which may not be present + if colnames is None: + colnames = ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"] + cols_to_clean = [ - c for c in ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"] if + c for c in colnames if c in data_to_clean.columns ] @@ -492,6 +496,8 @@ class DataProcessor: for col in cols_to_clean: data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"], inplace=True) data_to_clean.drop(columns=[f"{col}_AVERAGE"], inplace=True) + # If we still have missings + data_to_clean[col].fillna(data_to_clean[col].mean(), inplace=True) return data_to_clean From 3d0ebbb24ee108a81fa83f6c9daeda4a30117091 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 14:42:16 +0000 Subject: [PATCH 53/75] completed build of new demo portfolio - some fixed required still --- backend/app/plan/router.py | 11 +++++++++-- datatypes/enums.py | 1 + recommendations/VentilationRecommendations.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4064452f..ff56aa38 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -73,7 +73,8 @@ async def trigger_plan(body: PlanTriggerRequest): input_properties = [] for config in plan_input: # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly - # TODO: implment validation + # TODO: implment validation. We should also standardise postcode and address in some fashion as + # a postcode of abcdef would be considered different to ABCDEF # Create a record in db property_id, is_new = create_property( session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] @@ -310,7 +311,8 @@ async def trigger_plan(body: PlanTriggerRequest): # 3) the recommendations logger.info("Uploading recommendations to the database") - for i in range(0, len(input_properties), BATCH_SIZE): + session.commit() + for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] @@ -323,6 +325,11 @@ async def trigger_plan(body: PlanTriggerRequest): ) create_property_details_epc(session, property_details_epc) + # TODO: TEMP + if p.data["uprn"] == "": + print("Get rid of me!") + p.data["uprn"] = 0 + property_data = p.get_full_property_data() update_property_data( session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data diff --git a/datatypes/enums.py b/datatypes/enums.py index 1b0959e0..31f094ad 100644 --- a/datatypes/enums.py +++ b/datatypes/enums.py @@ -3,3 +3,4 @@ import enum class QuantityUnits(enum.Enum): m2 = "m2" + part = "part" diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 35de9b3b..a639905b 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -54,7 +54,7 @@ class VentilationRecommendations(Definitions): part[0]["estimated_cost"] = estimated_cost part[0]["quantity"] = n_units - part[0]["quantity_unit"] = None + part[0]["quantity_unit"] = "part" # We recommend installing two mechanical ventilation systems self.recommendation = [ From 2cc5a8f4652d9a06b3f51b8cd65a9633bb47aa86 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 16:22:51 +0000 Subject: [PATCH 54/75] Adding the clear_portfolio method for quickly emptying out a portfolio for easy rebuilds --- .../db/functions/recommendations_functions.py | 53 ++++++++++++++++--- backend/app/plan/router.py | 2 +- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index b9ec6fc3..0bdf69ce 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -1,10 +1,14 @@ -from sqlalchemy import insert +from sqlalchemy import insert, delete +from sqlalchemy.orm import Session from backend.app.db.models.recommendations import Plan, Recommendation, RecommendationMaterials, PlanRecommendations +from backend.app.db.models.portfolio import PropertyModel, PropertyTargetsModel, PropertyDetailsMeter, \ + PropertyDetailsEpcModel -def create_plan(session, plan): +def create_plan(session: Session, plan): """ This function will create a record for the plan in the database if it does not exist. + :param session: The database session :param plan: dictionary of data representing a plan to be created """ @@ -15,7 +19,7 @@ def create_plan(session, plan): return new_plan.id -def create_recommendation(session, recommendation): +def create_recommendation(session: Session, recommendation): """ This function will create a record for the recommendation in the database if it does not exist. :param session: The database session @@ -29,7 +33,7 @@ def create_recommendation(session, recommendation): return new_recommendation.id -def create_recommendation_material(session, recommendation_id, material_id, depth): +def create_recommendation_material(session: Session, recommendation_id, material_id, depth): """ This function will create a record for the recommendation_material in the database if it does not exist. :param session: The databse session @@ -49,9 +53,10 @@ def create_recommendation_material(session, recommendation_id, material_id, dept return new_recommendation_material.id -def create_plan_recommendations(session, plan_id, recommendation_ids): +def create_plan_recommendations(session: Session, plan_id, recommendation_ids): """ This function will create records for the plan_recommendation in the database. + :param session: The database session :param plan_id: ID of the plan :param recommendation_ids: list of recommendation IDs """ @@ -63,7 +68,7 @@ def create_plan_recommendations(session, plan_id, recommendation_ids): session.execute(insert(PlanRecommendations).values(data)) -def upload_recommendations(session, recommendations_to_upload, property_id): +def upload_recommendations(session: Session, recommendations_to_upload, property_id): # Prepare data for bulk insert for Recommendation recommendations_data = [ { @@ -112,3 +117,39 @@ def upload_recommendations(session, recommendations_to_upload, property_id): session.flush() return uploaded_recommendation_ids + + +def clear_portfolio(session: Session, portfolio_id: int): + # Fetch all property IDs associated with the given portfolio + property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all() + property_ids = [p.id for p in property_ids] + + # Fetch all recommendation IDs associated with the properties + recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all() + recommendation_ids = [r.id for r in recommendation_ids] + + # Delete all entries from RecommendationMaterials for these recommendations + session.execute( + delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) + ) + + # Delete all entries from PlanRecommendations that reference plans in the portfolio + session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_( + session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar() + ))) + + # Delete all Plans associated with the portfolio + session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id)) + + # Delete all Recommendations associated with the properties + session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids))) + + # Now, delete the PropertyModels and related details + # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel + session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id)) + # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids))) + session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)) + session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id)) + + # Commit the changes + session.commit() diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index ff56aa38..83a57d07 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -312,7 +312,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Uploading recommendations to the database") session.commit() - for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): + for i in range(0, len(input_properties), BATCH_SIZE): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] From a74459bf46b171f37542af3e60ab3498a3822f89 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 11:53:22 +0000 Subject: [PATCH 55/75] Creating new cost class to handle new costing data --- backend/app/plan/utils.py | 2 +- recommendations/Costs.py | 139 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 recommendations/Costs.py diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 36e90d61..e2bf9d86 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -173,7 +173,7 @@ def create_recommendation_scoring_data( parts = recommendation["parts"] if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - + scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: diff --git a/recommendations/Costs.py b/recommendations/Costs.py new file mode 100644 index 00000000..efa22fd8 --- /dev/null +++ b/recommendations/Costs.py @@ -0,0 +1,139 @@ +class Costs: + """ + A class to calculate the costs associated with construction works, + specifically focusing on cavity wall insulation. + It includes contingency, preliminaries, profit margin, and VAT calculations. + + As a sense check, there is a useful article from checkatrade on retrofitting walls and expected costs: + https://www.checkatrade.com/blog/cost-guides/retrofit-insulation-cost/ + """ + + # Contingency is a percentage of the total cost of the work and covers unforseen expenses + # We assume a conservative 10% contingency for all works which is a rate defined by SPONs + CONTINGENCY = 0.1 + + # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs + # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% + # rate, on the total cost before VAT, as recommended by SPONs + PRELIMINARIES = 0.1 + + VAT_RATE = 0.2 + PROFIT_MARGIN = 0.15 + + def __init__(self, property_instance): + """ + Initializes the Costs class with a property instance. + + :param property_instance: Instance of a Property class containing relevant details like wall area. + """ + if not hasattr(property_instance, 'insulation_wall_area'): + raise ValueError("Property instance must have an 'insulation_wall_area' attribute") + self.property = property_instance + + def cavity_wall_insulation(self, material): + """ + Calculates the total cost for cavity wall insulation based on material and labor costs, + including contingency, preliminaries, profit, and VAT. + + :return: A dictionary containing detailed cost breakdown. + """ + # Cost per m2 + material = { + "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " + "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " + "retaining discs", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 2.25, + "labour_hours": 0.13 + } + + material_cost_per_m2 = material["material_cost"] + wall_area = self.property.insulation_wall_area + + # This is the amount of material required in m3, assuming a standard 75mm depth + volume = 0.075 * wall_area + + base_material_cost = material_cost_per_m2 * wall_area + labour_cost = material["labour_cost"] * wall_area + + subtotal_before_profit = base_material_cost + labour_cost + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + labour_hours = material["labour_hours"] * wall_area + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": base_material_cost, + "profit": profit_cost, + "labour_hours": labour_hours + } + + def cavity_wall_insulation(self, material): + """ + Calculates the total cost for cavity wall insulation based on material and labor costs, + including contingency, preliminaries, profit, and VAT. + + :return: A dictionary containing detailed cost breakdown. + """ + # Cost per m2 + material = { + "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " + "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " + "retaining discs", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 2.25, + "labour_hours": 0.13 + } + + material_cost_per_m2 = material["material_cost"] + wall_area = self.property.insulation_wall_area + + # This is the amount of material required in m3, assuming a standard 75mm depth + volume = 0.075 * wall_area + + base_material_cost = material_cost_per_m2 * wall_area + labour_cost = material["labour_cost"] * wall_area + + subtotal_before_profit = base_material_cost + labour_cost + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + labour_hours = material["labour_hours"] * wall_area + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": base_material_cost, + "profit": profit_cost, + "labour_hours": labour_hours + } From 89c75dcd5a2e3f1bf0d097cf7c1b36a024e56c32 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 11:56:03 +0000 Subject: [PATCH 56/75] Added in loft insulation method --- recommendations/Costs.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index efa22fd8..6dc2e7aa 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -84,7 +84,7 @@ class Costs: "labour_hours": labour_hours } - def cavity_wall_insulation(self, material): + def loft_insulation(self, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -92,26 +92,21 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 - material = { - "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " - "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " - "retaining discs", - "depth": 75, - "thermal_conductivity": 0.037, - "prime_cost": 5.17, - "material_cost": 5.62, - "labour_cost": 2.25, - "labour_hours": 0.13 - } + # material = { + # "description": "Crown Loft Roll 44 glass fibre roll", + # "depth": 270, + # "thermal_conductivity": 0.044, + # "prime_cost": None, + # "material_cost": 5.91938, + # "labour_cost": 1.96, + # "labour_hours": 0.11 + # } material_cost_per_m2 = material["material_cost"] - wall_area = self.property.insulation_wall_area + floor_area = self.property.floor_area - # This is the amount of material required in m3, assuming a standard 75mm depth - volume = 0.075 * wall_area - - base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area + base_material_cost = material_cost_per_m2 * floor_area + labour_cost = material["labour_cost"] * floor_area subtotal_before_profit = base_material_cost + labour_cost @@ -125,7 +120,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * wall_area + labour_hours = material["labour_hours"] * floor_area return { "total": total_cost, From 4848c2f9c1d72cdc6b45a66fe83f6e1418fd077b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 17:33:55 +0000 Subject: [PATCH 57/75] implemented internal_wall_insulation cost method --- recommendations/Costs.py | 224 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 213 insertions(+), 11 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 6dc2e7aa..b78d8627 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,3 +1,6 @@ +import numpy as np + + class Costs: """ A class to calculate the costs associated with construction works, @@ -38,17 +41,19 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 - material = { - "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " - "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " - "retaining discs", - "depth": 75, - "thermal_conductivity": 0.037, - "prime_cost": 5.17, - "material_cost": 5.62, - "labour_cost": 2.25, - "labour_hours": 0.13 - } + # material = { + # "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or + # other " + # "equal; as full or partial cavity fill; including cutting and fitting around wall ties + # and " + # "retaining discs", + # "depth": 75, + # "thermal_conductivity": 0.037, + # "prime_cost": 5.17, + # "material_cost": 5.62, + # "labour_cost": 2.25, + # "labour_hours": 0.13 + # } material_cost_per_m2 = material["material_cost"] wall_area = self.property.insulation_wall_area @@ -132,3 +137,200 @@ class Costs: "profit": profit_cost, "labour_hours": labour_hours } + + def internal_wall_insulation(self, material, non_insulation_materials): + """ + Broadly speaking, the high level steps to an internal wall insulation job are the following: + + 1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles. + It's important to factor in the disposal of debris and the potential need for additional protective + measures to ensure the safety of the work area. + + 2) Insulation Installation: This is the core part of the process where the chosen insulation material is + applied. The choice of insulation material will depend on several factors including thermal performance, + wall construction, and space constraints. + + 3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation, + which can compromise its effectiveness and lead to mold growth. + + 4) Re-decoration: This involves applying plaster to the wall and then painting. + The quality of finish here is important for both aesthetic and functional reasons. + + 5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames, + or window sills might be necessary. + :return: + """ + + # Parsing the provided table into a list of dictionaries + + # non_insulation_materials = [ + # {'type': 'iwi_wall_demolition', + # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, + # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, + # {'type': 'iwi_wall_demolition', + # 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, + # 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, + # {'type': 'iwi_wall_demolition', + # 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' + # 'plaster', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, + # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Visqueen High Performance Vapour Barrier', + # 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, + # 'link': 'SPONs', + # 'material_cost': 1.21, + # 'plant_cost': 0, + # 'prime_material_cost': 0.58, + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 1.69, + # 'type': 'iwi_vapour_barrier'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' + # 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', + # 'labour_cost': 6.58, + # 'labour_hours_per_unit': 0.25, + # 'link': "", + # 'material_cost': 0.06, + # 'plant_cost': 0, + # 'prime_material_cost': 0.0, + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 6.64, + # 'type': 'iwi_redecoration'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' + # '5m high', + # 'labour_cost': 0.0, + # 'labour_hours_per_unit': 0.21, + # 'link': "", + # 'material_cost': 0.41, + # 'plant_cost': 0, + # 'prime_material_cost': "", + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 4.34, + # 'type': 'iwi_redecoration'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Fitting existing softwood skirting or architrave to new ' + # 'frames; 150mm high', + # 'labour_cost': 4.87, + # 'labour_hours_per_unit': 0.01, + # 'link': "", + # 'material_cost': 4.86, + # 'plant_cost': 0, + # 'prime_material_cost': "", + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 4.88, + # 'type': 'iwi_finishes'} + # ] + # + # material = { + # "type": "internal_wall_insulation", + # "description": "Ecotherm Eco-Versal PIR Insulation Board", + # "depth": 150, + # "depth_unit": "mm", + # "cost_unit": "gbp_per_m2", + # "thermal_conductivity": 0.022, + # "thermal_conductivity_unit": "watt_per_meter_kelvin", + # "prime_material_cost": "", + # "material_cost": 11.68, + # "labour_cost": 3.12, + # "labour_hours_per_unit": 0.18, + # "plant_cost": "", + # "total_cost": 14.8, + # "link": "SPONs" + # } + + # Cost per m2 + wall_area = self.property.insulation_wall_area + + # Extract and check the different types of data we'll need + demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] + finishes_data = [x for x in non_insulation_materials if x["type"] == "iwi_finishes"] + if not demolition_data: + raise ValueError("No data found for iwi_wall_demolition") + + if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2) or (len(finishes_data) != 1): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + # Since we don't know the exact wall construction, we take an average for demolition costs, since + # the cost will depend on the type of wall construction + demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * wall_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area + redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) + finishes_material_costs = finishes_data[0]["material_cost"] * wall_area + + demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) + + demolition_labour_costs = sum([x["labour_cost"] * wall_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * wall_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area + redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) + finishes_labour_costs = finishes_data[0]["labour_cost"] * wall_area + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs + finishes_labour_costs) + + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs + finishes_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * wall_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) + finishes_labour_hours = finishes_data[0]["labour_hours_per_unit"] * wall_area + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours + finishes_labour_hours) + + # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people + labour_days = (labour_hours / 8) / 5 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From 20bc4cab0e527bec5821a0b96f10c4bf3a253959 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 17:59:53 +0000 Subject: [PATCH 58/75] Added wip labour adjustment table --- recommendations/Costs.py | 64 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index b78d8627..fafc15aa 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,5 +1,53 @@ import numpy as np +# Example - delete me +from backend.Property import Property + +print("DELETE ME IN COSTS CLASS") +from epc_api.client import EpcClient + +epc_client = EpcClient(auth_token=AUTH_TOKEN) +p1 = Property( + postcode="NN1 5JY", + address1="2 South Terrace", + epc_client=epc_client, + id=0 +) + +p2 = Property( + postcode="PO12 4TY", + address1="25 Albert Street", + epc_client=epc_client, + id=0 +) + +p1.search_address_epc() +p2.search_address_epc() + +p1.set_basic_property_dimensions() +p2.set_basic_property_dimensions() + +regional_labour_variations = [ + {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, + {"Region": "Inner London", "Adjustment_Factor": 1.05}, + {"Region": "South East", "Adjustment_Factor": 0.96}, + {"Region": "South West", "Adjustment_Factor": 0.90}, + {"Region": "East of England", "Adjustment_Factor": 0.93}, + {"Region": "East Midlands", "Adjustment_Factor": 0.88}, + {"Region": "West Midlands", "Adjustment_Factor": 0.87}, + {"Region": "North East", "Adjustment_Factor": 0.83}, + {"Region": "North West", "Adjustment_Factor": 0.88}, + {"Region": "Yorkshire and Humberside", "Adjustment_Factor": 0.86}, + {"Region": "Wales", "Adjustment_Factor": 0.88}, + {"Region": "Scotland", "Adjustment_Factor": 0.88}, + {"Region": "Northern Ireland", "Adjustment_Factor": 0.76} +] + +county_map = { + "Northamptonshire": "East Midlands", + "Hampshire": "South East", +} + class Costs: """ @@ -32,6 +80,16 @@ class Costs: if not hasattr(property_instance, 'insulation_wall_area'): raise ValueError("Property instance must have an 'insulation_wall_area' attribute") self.property = property_instance + self.regional_labour_variations = regional_labour_variations + + self.county = county_map.get(self.property.data["county"], None) + if self.county is None: + raise ValueError("County not found in county map") + + self.labour_adjustment_factor = [ + x["Adjustment_Factor"] for x in self.regional_labour_variations if + x["Region"] == self.county + ][0] def cavity_wall_insulation(self, material): """ @@ -62,7 +120,7 @@ class Costs: volume = 0.075 * wall_area base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area + labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor subtotal_before_profit = base_material_cost + labour_cost @@ -111,7 +169,7 @@ class Costs: floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area - labour_cost = material["labour_cost"] * floor_area + labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor subtotal_before_profit = base_material_cost + labour_cost @@ -296,6 +354,8 @@ class Costs: labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + redecoration_labour_costs + finishes_labour_costs) + labour_costs = labour_costs * self.labour_adjustment_factor + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + redecoration_material_costs + finishes_material_costs) From 5aebd43c4b34bf04083153036fc463195d6b49f6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 18:02:18 +0000 Subject: [PATCH 59/75] added ValueError for no labour adjustment factor found --- recommendations/Costs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index fafc15aa..c855e690 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -91,6 +91,9 @@ class Costs: x["Region"] == self.county ][0] + if not self.labour_adjustment_factor: + raise ValueError("Labour adjustment factor not found") + def cavity_wall_insulation(self, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, From 9adfa4c07538bd03228cf6cdc991c1c9b5f93b92 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 22:29:55 +0000 Subject: [PATCH 60/75] implemented suspended floor insulation --- backend/Property.py | 5 +- recommendations/Costs.py | 120 ++++++++++++++++++++++++ recommendations/recommendation_utils.py | 19 ---- 3 files changed, 122 insertions(+), 22 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 370eca06..4106c60a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1,5 +1,4 @@ from datetime import datetime -from collections import Counter import re import os import pandas as pd @@ -13,7 +12,7 @@ from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup from recommendations.recommendation_utils import ( - estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area + estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area ) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') @@ -596,7 +595,7 @@ class Property(Definitions): self.number_of_rooms = float(self.data["number-habitable-rooms"]) if self.data["property-type"] == "House": - self.number_of_floors = estimate_floors(self.floor_area, self.number_of_rooms) + self.number_of_floors = 2 elif self.data["property-type"] in ["Flat", "Bungalow"]: self.number_of_floors = 1 elif self.data["property-type"] == "Maisonette": diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c855e690..49298d5f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -27,6 +27,12 @@ p2.search_address_epc() p1.set_basic_property_dimensions() p2.set_basic_property_dimensions() +import pandas as pd + +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - suspended_floor_insulation.csv") +df = df.to_dict("records") + +# This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, {"Region": "Inner London", "Adjustment_Factor": 1.05}, @@ -397,3 +403,117 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def suspended_floor_insulation(self, material, non_insulation_materials): + """ + We characterise the steps for suspended floor insulation as the following tasks: + + 1) Removal of Carpet and Underfelt: Where necessary, remove existing floor coverings to access the floorboards. + 2) Removal of Floor Boarding: Carefully remove floorboards to access the space beneath for insulation. + 3) Installation of Vapour Barrier: Install a vapour barrier to prevent moisture from affecting + the insulation and floor structure. + 4) Installation of Insulation: Fit the chosen insulation material between the joists in the floor void. + 5) Refixing Floorboards: Replace and secure the floorboards after insulation installation. + 6) Re-carpeting: Lay down the carpet or other floor coverings once the insulation and floorboards are in place. + :return: + """ + + # material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0, + # 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + # 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0, + # 'total_cost': 13.46, 'link': 'SPONs', + # 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. We use + # the ' + # 'same values as in Crown loft roll 44, since it is also an insulation roll'} + # + # non_insulation_materials = [ + # {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + # 'therefore there is no need for a skip'}, + # {'type': 'suspended_floor_demolition', + # 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, + # 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, + # 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + # 'therefore we need just labour rates'}] + + insulation_floor_area = self.property.floor_area / self.property.number_of_floors + + demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"] + + if (len(demolition_data) != 2) or (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + demolition_material_costs = sum([x["material_cost"] * insulation_floor_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * insulation_floor_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area + redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data]) + + demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * insulation_floor_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area + redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data]) + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data]) + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours) + + # Assume a team of 3 people for a small to medium size project + labour_days = (labour_hours / 8) / 3 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 063a274c..7cba8257 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -527,25 +527,6 @@ def get_wall_type( return None -def estimate_floors(floor_area, num_rooms): - """ - Simple utility funciton, which assuming a 15m squared room, estimates the number of floors in a property - :param floor_area: Gross floor area of a property - :param num_rooms: Number of rooms in a property - :return: Number of floors in a property - """ - # Estimate total room area - total_room_area = num_rooms * 15 - - # Estimate the number of floors - floors = floor_area / total_room_area - - # Round up to the nearest whole number - floors = round(floors) - - return floors - - def estimate_wall_area(num_floors, floor_height, perimeter): wall_area_one_floor = perimeter * floor_height From dea9a7fb17f28f21bb545666aa68a5bde36184ac Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 08:47:37 +0000 Subject: [PATCH 61/75] implemented solid floor insulation --- recommendations/Costs.py | 205 +++++++++++++++++++++++++++++++++++---- 1 file changed, 187 insertions(+), 18 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 49298d5f..1c8cf6d8 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -29,7 +29,7 @@ p2.set_basic_property_dimensions() import pandas as pd -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - suspended_floor_insulation.csv") +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - solid_floor_insulation.csv") df = df.to_dict("records") # This data comes from SPONs @@ -69,6 +69,9 @@ class Costs: # We assume a conservative 10% contingency for all works which is a rate defined by SPONs CONTINGENCY = 0.1 + # Where there is more uncertainty, a higher contingency rate is used + HIGH_RISH_CONTINGENCY = 0.15 + # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% # rate, on the total cost before VAT, as recommended by SPONs @@ -100,7 +103,7 @@ class Costs: if not self.labour_adjustment_factor: raise ValueError("Labour adjustment factor not found") - def cavity_wall_insulation(self, material): + def cavity_wall_insulation(self, wall_area, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -123,7 +126,7 @@ class Costs: # } material_cost_per_m2 = material["material_cost"] - wall_area = self.property.insulation_wall_area + # wall_area = self.property.insulation_wall_area # This is the amount of material required in m3, assuming a standard 75mm depth volume = 0.075 * wall_area @@ -156,7 +159,7 @@ class Costs: "labour_hours": labour_hours } - def loft_insulation(self, material): + def loft_insulation(self, floor_area, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -175,7 +178,7 @@ class Costs: # } material_cost_per_m2 = material["material_cost"] - floor_area = self.property.floor_area + # floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor @@ -205,7 +208,7 @@ class Costs: "labour_hours": labour_hours } - def internal_wall_insulation(self, material, non_insulation_materials): + def internal_wall_insulation(self, wall_area, material, non_insulation_materials): """ Broadly speaking, the high level steps to an internal wall insulation job are the following: @@ -309,7 +312,7 @@ class Costs: # 'thermal_conductivity': "", # 'thermal_conductivity_unit': "", # 'total_cost': 4.88, - # 'type': 'iwi_finishes'} + # 'type': 'iwi_redecoration'} # ] # # material = { @@ -330,17 +333,16 @@ class Costs: # } # Cost per m2 - wall_area = self.property.insulation_wall_area + # wall_area = self.property.insulation_wall_area # Extract and check the different types of data we'll need demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] - finishes_data = [x for x in non_insulation_materials if x["type"] == "iwi_finishes"] if not demolition_data: raise ValueError("No data found for iwi_wall_demolition") - if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2) or (len(finishes_data) != 1): + if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3): raise ValueError("Incorrect number of data entries for non-insulation materials") # Break out the individual material costs @@ -350,7 +352,6 @@ class Costs: insulation_material_costs = material["material_cost"] * wall_area vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) - finishes_material_costs = finishes_data[0]["material_cost"] * wall_area demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) @@ -358,15 +359,14 @@ class Costs: insulation_labour_costs = material["labour_cost"] * wall_area vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) - finishes_labour_costs = finishes_data[0]["labour_cost"] * wall_area labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + - redecoration_labour_costs + finishes_labour_costs) + redecoration_labour_costs) labour_costs = labour_costs * self.labour_adjustment_factor materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + - redecoration_material_costs + finishes_material_costs) + redecoration_material_costs) subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs @@ -384,10 +384,9 @@ class Costs: insulation_labour_hours = material["labour_hours_per_unit"] * wall_area vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) - finishes_labour_hours = finishes_data[0]["labour_hours_per_unit"] * wall_area labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + - redecoration_labour_hours + finishes_labour_hours) + redecoration_labour_hours) # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people labour_days = (labour_hours / 8) / 5 @@ -404,7 +403,7 @@ class Costs: "labour_days": labour_days } - def suspended_floor_insulation(self, material, non_insulation_materials): + def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): """ We characterise the steps for suspended floor insulation as the following tasks: @@ -455,7 +454,7 @@ class Costs: # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' # 'therefore we need just labour rates'}] - insulation_floor_area = self.property.floor_area / self.property.number_of_floors + # insulation_floor_area = self.property.floor_area / self.property.number_of_floors demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] @@ -517,3 +516,173 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): + """ + We characterise the steps for solid floor insulation as the following tasks: + + 1) Removal of Carpet and Underfelt: This is the initial stage where any existing floor coverings, like carpets, + tiles, or linoleum, are carefully removed. This exposes the solid floor beneath, which is typically concrete. + + 2) Preparation of Flooring: This step is critical. It involves: + - Cleaning the existing floor surface thoroughly to remove debris and ensure a flat surface. + - Assessing and repairing any damage to the concrete floor. This might include filling cracks or leveling + uneven areas. + + 3) Installation of a Damp Proof Membrane (DPM): Before installing insulation, a DPM is often laid down to + prevent moisture from rising into the insulation and the interior space. This step is crucial in areas prone to + dampness. + + 4) Install Insulation: The insulation, often in the form of rigid foam boards, is laid over the DPM. + The choice of insulation material will depend on the desired thermal properties and the available floor height. + Care is taken to minimize thermal bridges and ensure a snug fit between insulation boards. + + 5) Laying a New Subfloor: Over the insulation, a new subfloor is often installed. This could be a layer of + screed (a type of concrete) or wooden boarding, depending on the specific requirements and preferences. + + 6) Re-decoration and Finishing Touches: Once the subfloor is in place and has set or dried (if necessary), + the final floor finish can be applied. This might involve: + - Laying new tiles, wooden flooring, or other chosen materials. + - If you're planning to re-carpet, this would be the stage to do it. + - Skirting boards may need to be refitted or replaced. + + 7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door + thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments. + :param material: + :param non_insulation_materials: + :return: + """ + + material = { + 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + } + + non_insulation_materials = [ + {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'solid_floor_preparation', + 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + 'description': 'Clean out crack to ' + 'form a 20mm×20mm ' + 'groove and fill with ' + 'cement: mortar mixed ' + 'with bonding agent', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, + 'material_cost': 6.91, + 'labour_cost': 18.99, + 'labour_hours_per_unit': 0.61, + 'plant_cost': 0.16, + 'total_cost': 26.06, 'link': 0, + 'Notes': 'This step is the ' + 'assessment and repair of ' + 'any damage to the concrete ' + 'floor such as filling ' + 'cracks or levelling uneven ' + 'areas'}, + {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'solid_floor_redecoration', + 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + 'Notes': 'This is the screed layer, placed on top of the insulation'}, + {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}, + {'type': 'solid_floor_redecoration', + 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + ] + + # insulation_floor_area = self.property.floor_area / self.property.number_of_floors + + demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"] + preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_redecoration"] + + if ((len(demolition_data) != 1) or (len(preparation_data) != 2) or (len(vapour_barrier_data) != 1) or + (len(redecoration_data) != 3)): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + preparation_material_costs = sum([x["material_cost"] * insulation_floor_area for x in preparation_data]) + insulation_material_costs = material["material_cost"] * insulation_floor_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area + redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data]) + + demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data]) + preparation_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in preparation_data]) + insulation_labour_costs = material["labour_cost"] * insulation_floor_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area + redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data]) + + preparation_plant_costs = sum([x["plant_cost"] * insulation_floor_area for x in preparation_data]) + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs + preparation_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_cost = (preparation_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_cost + preparation_plant_costs + + # We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors, + # as well as scope for damage to the existing floor during preparation. + contingency_cost = subtotal_before_profit * self.HIGH_RISH_CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + vat_cost = subtotal_before_vat * self.VAT_RATE + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data]) + preparation_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in preparation_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data]) + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours + preparation_labour_hours) + + # Assume a team of 3 people for a small to medium size project + labour_days = (labour_hours / 8) / 3 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_cost, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From 63de7c19df65dfdc58f51dd3a62230ca95cf8129 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 14:51:30 +0000 Subject: [PATCH 62/75] completed pricing for ewi --- recommendations/Costs.py | 348 +++++++++++++++++++++++++++++++-------- 1 file changed, 278 insertions(+), 70 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 1c8cf6d8..c6e4ceb7 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -29,7 +29,7 @@ p2.set_basic_property_dimensions() import pandas as pd -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - solid_floor_insulation.csv") +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - external_wall_insulation.csv") df = df.to_dict("records") # This data comes from SPONs @@ -61,8 +61,11 @@ class Costs: specifically focusing on cavity wall insulation. It includes contingency, preliminaries, profit margin, and VAT calculations. - As a sense check, there is a useful article from checkatrade on retrofitting walls and expected costs: + As a sense check, there is a useful article from checkatrade on retrofitting and expected costs: https://www.checkatrade.com/blog/cost-guides/retrofit-insulation-cost/ + + Another useful article for benchmarking the cost of floor insulation: + https://www.checkatrade.com/blog/cost-guides/floor-insulation-cost/ """ # Contingency is a percentage of the total cost of the work and covers unforseen expenses @@ -70,13 +73,22 @@ class Costs: CONTINGENCY = 0.1 # Where there is more uncertainty, a higher contingency rate is used - HIGH_RISH_CONTINGENCY = 0.15 + HIGH_RISK_CONTINGENCY = 0.15 + # When there is less uncertainty, a lower contingency rate is used + LOW_RISK_CONTINGENCY = 0.05 # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% # rate, on the total cost before VAT, as recommended by SPONs PRELIMINARIES = 0.1 + # For higher risk projects, a higher preliminaries rate is used. SPONs indicates that a higher risk project might + # have a preliminaries of 12-14% so we use 12% as the median for the preliminaries rate. + # For External wall insulation (EWI), we use 15% as the preliminaries rate if we think the property might + # need scaffolding, otherwise we use 12%. This is to account for any site preparation that might be required + EWI_NO_SCAFFOLDING_PRELIMINARIES = 0.12 + EWI_SCAFFOLDING_PRELIMINARIES = 0.15 + VAT_RATE = 0.2 PROFIT_MARGIN = 0.15 @@ -355,7 +367,8 @@ class Costs: demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - demolition_labour_costs = sum([x["labour_cost"] * wall_area for x in demolition_data]) + # Again for demolition, we average since we aren't sure which demolition process will be used + demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) insulation_labour_costs = material["labour_cost"] * wall_area vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) @@ -380,7 +393,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - demolition_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) insulation_labour_hours = material["labour_hours_per_unit"] * wall_area vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) @@ -553,70 +566,70 @@ class Costs: :return: """ - material = { - 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, - 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 - } - - non_insulation_materials = [ - {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - 'therefore there is no need for a skip'}, - {'type': 'solid_floor_preparation', - 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', - 'description': 'Clean out crack to ' - 'form a 20mm×20mm ' - 'groove and fill with ' - 'cement: mortar mixed ' - 'with bonding agent', - 'depth': 0, 'depth_unit': 0, - 'cost_unit': 0, - 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, - 'material_cost': 6.91, - 'labour_cost': 18.99, - 'labour_hours_per_unit': 0.61, - 'plant_cost': 0.16, - 'total_cost': 26.06, 'link': 0, - 'Notes': 'This step is the ' - 'assessment and repair of ' - 'any damage to the concrete ' - 'floor such as filling ' - 'cracks or levelling uneven ' - 'areas'}, - {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - {'type': 'solid_floor_redecoration', - 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, - 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', - 'Notes': 'This is the screed layer, placed on top of the insulation'}, - {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - 'therefore we need just labour rates'}, - {'type': 'solid_floor_redecoration', - 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} - ] + # material = { + # 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + # 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + # 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + # 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + # } + # + # non_insulation_materials = [ + # {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + # 'therefore there is no need for a skip'}, + # {'type': 'solid_floor_preparation', + # 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + # 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + # 'description': 'Clean out crack to ' + # 'form a 20mm×20mm ' + # 'groove and fill with ' + # 'cement: mortar mixed ' + # 'with bonding agent', + # 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, + # 'material_cost': 6.91, + # 'labour_cost': 18.99, + # 'labour_hours_per_unit': 0.61, + # 'plant_cost': 0.16, + # 'total_cost': 26.06, 'link': 0, + # 'Notes': 'This step is the ' + # 'assessment and repair of ' + # 'any damage to the concrete ' + # 'floor such as filling ' + # 'cracks or levelling uneven ' + # 'areas'}, + # {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'solid_floor_redecoration', + # 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + # 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + # 'Notes': 'This is the screed layer, placed on top of the insulation'}, + # {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + # 'therefore we need just labour rates'}, + # {'type': 'solid_floor_redecoration', + # 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + # 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + # ] # insulation_floor_area = self.property.floor_area / self.property.number_of_floors @@ -655,7 +668,7 @@ class Costs: # We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors, # as well as scope for damage to the existing floor during preparation. - contingency_cost = subtotal_before_profit * self.HIGH_RISH_CONTINGENCY + contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES profit_cost = subtotal_before_profit * self.PROFIT_MARGIN @@ -686,3 +699,198 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def external_wall_insulation(self, wall_area, material, non_insulation_materials): + """ + We characterise external wall insulation as the following steps: + + 1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective + sheets to protect the flooring and landscaping around the work area. + + 2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or + detached houses. For terraced houses or lower-level work, scaffolding might not be necessary. + + 3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material, + and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement + of the top surface may be necessary. + + 4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation + boards and stabilize the wall surface, especially if it's old or weathered. + + 5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings, + or a combination of both. + + 6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation. + This layer provides strength and helps prevent cracking. + + 7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation + and provides an aesthetic look. + + 8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that + were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness. + + 9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system, + followed by cleaning up the site to remove all debris and materials. + + In the actual materials data, at this point, we have costing for: + - wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition) + - wall surface cleaning and priming (ewi_wall_preparation) + - insulation (external_wall_insulation) + - basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh) + + All of this data comes from SPONS, however there are some clear features missing. Because we could not find + suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding, + re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by + increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these + steps should be included in the materials data. We will look to improve this in the future, with data from + installers + + :param wall_area: + :param material: + :param non_insulation_materials: + :return: + """ + + # For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding + + if self.property.data["property-type"] == "House": + if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]: + preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES + else: + preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES + elif self.property.data["property-type"] == "Maisonette": + preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES + elif self.property.data["property-type"] == "Bungalow": + preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES + else: + raise ValueError("Unsupported property type - haven't handled flats") + + # non_insulation_materials = [x for x in df if x["type"] != "external_wall_insulation"] + # insulation_materials = [x for x in df if x["type"] == "external_wall_insulation"] + # material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', + # 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, + # 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, + # 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} + # non_insulation_materials = [ + # {'type': 'ewi_wall_demolition', + # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' + # 'hammer; plaster to walls.', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, + # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, + # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', + # 'description': 'Stud walls: Remove wall linings ' + # 'including battening behind; ' + # 'plasterboard and skim', + # 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, + # 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, + # 'plant_cost': 1.25, 'total_cost': 7.48, + # 'link': 'SPONs', 'Notes': 0}, + # {'type': 'ewi_wall_demolition', + # 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' + # 'behind; wood lath and plaster', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, + # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, + # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', + # 'description': 'Clean and prepare surfaces, ' + # 'one coat Keim dilution, ' + # 'one coat primer and two coats ' + # 'of Keim Ecosil paint; Brick or ' + # 'block walls; over 300 mm girth', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 7.3, + # 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, + # 'plant_cost': 0, 'total_cost': 12.92, + # 'link': 'SPONs', + # 'Notes': 'This work covers the preparation and ' + # 'priming of the wall before insulating'}, + # {'type': 'ewi_wall_redecoration', + # 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' + # 'separately) with horizontal PVC intermediate track and vertical ' + # 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' + # 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' + # 'Topcoat Render (white)', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, + # 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, + # 'total_cost': 69.94, 'link': 'SPONs', + # 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' + # 'cost of 99.17 per meter square. This includes the cost of insulation. ' + # 'To get the costing for just the works and not the insulation, ' + # 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' + # 'as an example, which costs £29.23 per meter square, giving us the cost ' + # 'of the remaining works without insulation. This material gives us a ' + # 'cost for basecoat, mesh application and a render finish'}] + + demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] + preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] + + if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + # Since we don't know the exact wall construction, we take an average for demolition costs, since + # the cost will depend on the type of wall construction + demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * wall_area + preparation_material_costs = preparation_data[0]["material_cost"] * wall_area + redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area + + demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) + + demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * wall_area + preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area + redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area + + labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs + + preparation_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * preliminaries_rate + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + vat_cost = subtotal_before_vat * self.VAT_RATE + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * wall_area + preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area + redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area + + labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours + + preparation_labour_hours) + + # Assume a team of 3-5 people for a small to medium size project + labour_days = (labour_hours / 8) / 4 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From a9a7d3f3a809fe40281cf732a26e2359fea9ae44 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:05:04 +0000 Subject: [PATCH 63/75] Added test_external_wall_area --- backend/Property.py | 9 ++++--- recommendations/Costs.py | 24 +++++++------------ recommendations/recommendation_utils.py | 23 ++++++++++++++++-- recommendations/tests/test_costs.py | 0 .../tests/test_recommendation_utils.py | 15 ++++++++++++ 5 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 recommendations/tests/test_costs.py diff --git a/backend/Property.py b/backend/Property.py index 4106c60a..8bdfa7e9 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -12,7 +12,7 @@ from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup from recommendations.recommendation_utils import ( - estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area + estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area ) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') @@ -612,8 +612,11 @@ class Property(Definitions): self.floor_area / self.number_of_floors, self.number_of_rooms / self.number_of_floors ) - self.insulation_wall_area = estimate_wall_area( - num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter + self.insulation_wall_area = estimate_external_wall_area( + num_floors=self.number_of_floors, + floor_height=self.floor_height, + perimeter=self.perimeter, + built_form=self.data["built-form"], ) self.pitched_roof_area = esimtate_pitched_roof_area( diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c6e4ceb7..d14b5f33 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -27,11 +27,6 @@ p2.search_address_epc() p1.set_basic_property_dimensions() p2.set_basic_property_dimensions() -import pandas as pd - -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - external_wall_insulation.csv") -df = df.to_dict("records") - # This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, @@ -120,28 +115,25 @@ class Costs: Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. + Because of some limitations in the SPONs data, there are no materials that can be blown through a wall, + therefore we have adapted similar materials, basing our estimates on 75mm cavity slabs, and have halved the + labour time required. That is why we still price based on wall area despite volume actually being the correct + metric. + :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 # material = { - # "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or - # other " - # "equal; as full or partial cavity fill; including cutting and fitting around wall ties - # and " - # "retaining discs", + # "description": "cwi", # "depth": 75, # "thermal_conductivity": 0.037, # "prime_cost": 5.17, # "material_cost": 5.62, - # "labour_cost": 2.25, - # "labour_hours": 0.13 + # "labour_cost": 1.125, + # "labour_hours": 0.065 # } material_cost_per_m2 = material["material_cost"] - # wall_area = self.property.insulation_wall_area - - # This is the amount of material required in m3, assuming a standard 75mm depth - volume = 0.075 * wall_area base_material_cost = material_cost_per_m2 * wall_area labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 7cba8257..217f313f 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -527,12 +527,31 @@ def get_wall_type( return None -def estimate_wall_area(num_floors, floor_height, perimeter): +def estimate_external_wall_area(num_floors, floor_height, perimeter, built_form): + """ + This method estimates the external wall area based on fundamental assumptions about the home + + + :param num_floors: Number of floors in the building. + :param floor_height: Height of one floor in meters. + :param perimeter: Total perimeter of the building on one floor in meters. + :param built_form: The built form of the property. This is used to determine the number of exposed walls. + :return: + """ wall_area_one_floor = perimeter * floor_height total_wall_area = wall_area_one_floor * num_floors - return total_wall_area + number_exposed_walls = { + 'End-Terrace': 3, + 'Mid-Terrace': 2, + 'Semi-Detached': 3, + 'Detached': 4, + } + + exposed_wall_area = total_wall_area * (number_exposed_walls[built_form] / 4) + + return exposed_wall_area def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py new file mode 100644 index 00000000..e69de29b diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 22280ed5..73796979 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -405,3 +405,18 @@ def test_esimtate_pitched_roof_area(): ) assert zero_roof_area2 == 0 + + +def test_external_wall_area(): + # Arrange: Define the test cases + test_cases = [ + (2, 3, 40, 'End-Terrace', 180), # 3 exposed walls + (2, 3, 40, 'Mid-Terrace', 120), # 2 exposed walls + (2, 3, 40, 'Semi-Detached', 180), # 3 exposed walls + (2, 3, 40, 'Detached', 240), # 4 exposed walls + ] + + # Act and Assert: Run the test cases + for num_floors, floor_height, perimeter, built_form, expected in test_cases: + result = recommendation_utils.estimate_external_wall_area(num_floors, floor_height, perimeter, built_form) + assert result == expected, f"Test failed for {built_form}: Expected {expected}, got {result}" From 824b92fedd908bb4dd78368f431bb9681816fb0e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:10:07 +0000 Subject: [PATCH 64/75] Added cavity wall test --- recommendations/Costs.py | 10 --------- recommendations/tests/test_costs.py | 33 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index d14b5f33..1fb1114d 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -122,16 +122,6 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # Cost per m2 - # material = { - # "description": "cwi", - # "depth": 75, - # "thermal_conductivity": 0.037, - # "prime_cost": 5.17, - # "material_cost": 5.62, - # "labour_cost": 1.125, - # "labour_hours": 0.065 - # } material_cost_per_m2 = material["material_cost"] diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index e69de29b..8ca12e0b 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -0,0 +1,33 @@ +from recommendations.Costs import Costs +from unittest.mock import Mock + + +class TestCosts: + + def test_cavity_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + + cwi_material = { + "description": "cwi", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 1.125, + "labour_hours": 0.065 + } + + cwi_results = costs.cavity_wall_insulation( + wall_area=95.9104281347967, + material=cwi_material, + ) + + assert cwi_results == {'total': 1027.0280465530302, 'subtotal': 855.8567054608585, 'vat': 171.1713410921717, + 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, + 'material': 539.0166061175574, 'profit': 95.09518949565093, + 'labour_hours': 6.234177828761786} From 11ba4c64d08efe5fbae89c3910fb79721ed59192 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:19:22 +0000 Subject: [PATCH 65/75] tested iwi costs --- backend/Property.py | 5 +- recommendations/Costs.py | 121 +------------------- recommendations/FloorRecommendations.py | 4 +- recommendations/RoofRecommendations.py | 1 + recommendations/tests/test_costs.py | 143 ++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 121 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 8bdfa7e9..a3328156 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -82,6 +82,7 @@ class Property(Definitions): self.insulation_wall_area = None self.floor_area = None self.pitched_roof_area = None + self.insulation_floor_area = None if epc_client: self.epc_client = epc_client @@ -619,8 +620,10 @@ class Property(Definitions): built_form=self.data["built-form"], ) + self.insulation_floor_area = self.floor_area / self.number_of_floors + self.pitched_roof_area = esimtate_pitched_roof_area( - floor_area=self.floor_area / self.number_of_floors, floor_height=self.floor_height + floor_area=self.insulation_floor_area, floor_height=self.floor_height ) def set_wall_type(self): diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 1fb1114d..e8f9d122 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -160,19 +160,7 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # Cost per m2 - # material = { - # "description": "Crown Loft Roll 44 glass fibre roll", - # "depth": 270, - # "thermal_conductivity": 0.044, - # "prime_cost": None, - # "material_cost": 5.91938, - # "labour_cost": 1.96, - # "labour_hours": 0.11 - # } - material_cost_per_m2 = material["material_cost"] - # floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor @@ -225,110 +213,6 @@ class Costs: :return: """ - # Parsing the provided table into a list of dictionaries - - # non_insulation_materials = [ - # {'type': 'iwi_wall_demolition', - # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, - # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, - # {'type': 'iwi_wall_demolition', - # 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, - # 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, - # {'type': 'iwi_wall_demolition', - # 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' - # 'plaster', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, - # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Visqueen High Performance Vapour Barrier', - # 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, - # 'link': 'SPONs', - # 'material_cost': 1.21, - # 'plant_cost': 0, - # 'prime_material_cost': 0.58, - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 1.69, - # 'type': 'iwi_vapour_barrier'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' - # 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', - # 'labour_cost': 6.58, - # 'labour_hours_per_unit': 0.25, - # 'link': "", - # 'material_cost': 0.06, - # 'plant_cost': 0, - # 'prime_material_cost': 0.0, - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 6.64, - # 'type': 'iwi_redecoration'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' - # '5m high', - # 'labour_cost': 0.0, - # 'labour_hours_per_unit': 0.21, - # 'link': "", - # 'material_cost': 0.41, - # 'plant_cost': 0, - # 'prime_material_cost': "", - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 4.34, - # 'type': 'iwi_redecoration'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Fitting existing softwood skirting or architrave to new ' - # 'frames; 150mm high', - # 'labour_cost': 4.87, - # 'labour_hours_per_unit': 0.01, - # 'link': "", - # 'material_cost': 4.86, - # 'plant_cost': 0, - # 'prime_material_cost': "", - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 4.88, - # 'type': 'iwi_redecoration'} - # ] - # - # material = { - # "type": "internal_wall_insulation", - # "description": "Ecotherm Eco-Versal PIR Insulation Board", - # "depth": 150, - # "depth_unit": "mm", - # "cost_unit": "gbp_per_m2", - # "thermal_conductivity": 0.022, - # "thermal_conductivity_unit": "watt_per_meter_kelvin", - # "prime_material_cost": "", - # "material_cost": 11.68, - # "labour_cost": 3.12, - # "labour_hours_per_unit": 0.18, - # "plant_cost": "", - # "total_cost": 14.8, - # "link": "SPONs" - # } - - # Cost per m2 - # wall_area = self.property.insulation_wall_area - # Extract and check the different types of data we'll need demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] @@ -365,7 +249,8 @@ class Costs: subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - contingency_cost = subtotal_before_profit * self.CONTINGENCY + # We use high risk contingency for iwi + contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES profit_cost = subtotal_before_profit * self.PROFIT_MARGIN @@ -384,7 +269,7 @@ class Costs: redecoration_labour_hours) # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people - labour_days = (labour_hours / 8) / 5 + labour_days = (labour_hours / 8) / 4 return { "total": total_cost, diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index bc24b6c3..5b194e0d 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -58,7 +58,7 @@ class FloorRecommendations(Definitions): ) property_type = self.property.data["property-type"] - floor_area = self.property.floor_area / self.property.number_of_floors + floor_area = self.property.insulation_floor_area year_built = self.property.year_built if self.property.floor["another_property_below"] | (self.property.floor["insulation_thickness"] in [ @@ -137,7 +137,7 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - quantity = self.property.floor_area / self.property.number_of_floors + quantity = self.property.insulation_floor_area estimated_cost = cost_per_unit * quantity diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index bfa63908..4f96f629 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -161,6 +161,7 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + # TODO: We should use the floor area divided by the number of floors to get the area of the roof estimated_cost = cost_per_unit * self.property.floor_area if roof["is_pitched"]: diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 8ca12e0b..8e2d76ce 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -31,3 +31,146 @@ class TestCosts: 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574, 'profit': 95.09518949565093, 'labour_hours': 6.234177828761786} + + def test_loft_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + loft_material = { + "description": "Crown Loft Roll 44 glass fibre roll", + "depth": 270, + "thermal_conductivity": 0.044, + "prime_cost": None, + "material_cost": 5.91938, + "labour_cost": 1.96, + "labour_hours": 0.11 + } + + loft_results = costs.loft_insulation( + floor_area=33.5, + material=loft_material, + ) + + assert loft_results == {'total': 414.8496486, 'subtotal': 345.70804050000004, 'vat': 69.14160810000001, + 'contingency': 25.608003000000004, 'preliminaries': 25.608003000000004, + 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685} + + def test_internal_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + iwi_non_insulation_materials = [ + {'type': 'iwi_wall_demolition', + 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, + 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, + {'type': 'iwi_wall_demolition', + 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, + 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, + {'type': 'iwi_wall_demolition', + 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' + 'plaster', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, + 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Visqueen High Performance Vapour Barrier', + 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, + 'link': 'SPONs', + 'material_cost': 1.21, + 'plant_cost': 0, + 'prime_material_cost': 0.58, + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 1.69, + 'type': 'iwi_vapour_barrier'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' + 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', + 'labour_cost': 6.58, + 'labour_hours_per_unit': 0.25, + 'link': "", + 'material_cost': 0.06, + 'plant_cost': 0, + 'prime_material_cost': 0.0, + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 6.64, + 'type': 'iwi_redecoration'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' + '5m high', + 'labour_cost': 0.0, + 'labour_hours_per_unit': 0.21, + 'link': "", + 'material_cost': 0.41, + 'plant_cost': 0, + 'prime_material_cost': "", + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 4.34, + 'type': 'iwi_redecoration'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Fitting existing softwood skirting or architrave to new ' + 'frames; 150mm high', + 'labour_cost': 4.87, + 'labour_hours_per_unit': 0.01, + 'link': "", + 'material_cost': 4.86, + 'plant_cost': 0, + 'prime_material_cost': "", + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 4.88, + 'type': 'iwi_redecoration'} + ] + + iwi_material = { + "type": "internal_wall_insulation", + "description": "Ecotherm Eco-Versal PIR Insulation Board", + "depth": 150, + "depth_unit": "mm", + "cost_unit": "gbp_per_m2", + "thermal_conductivity": 0.022, + "thermal_conductivity_unit": "watt_per_meter_kelvin", + "prime_material_cost": "", + "material_cost": 11.68, + "labour_cost": 3.12, + "labour_hours_per_unit": 0.18, + "plant_cost": "", + "total_cost": 14.8, + "link": "SPONs" + } + + iwi_results = costs.internal_wall_insulation( + wall_area=95.9104281347967, + material=iwi_material, + non_insulation_materials=iwi_non_insulation_materials + ) + + assert iwi_results == {'total': 6421.5484411659245, 'subtotal': 5351.29036763827, 'vat': 1070.258073527654, + 'contingency': 573.3525393898148, 'preliminaries': 382.2350262598765, + 'material': 1747.488000615996, 'profit': 573.3525393898148, + 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405} From ee698c40b5e8c562a21d6b3bf46def9262bf3833 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:30:38 +0000 Subject: [PATCH 66/75] Added unit tests for costs --- recommendations/Costs.py | 197 ++--------------------- recommendations/tests/test_costs.py | 241 +++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 184 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index e8f9d122..8f54a79a 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -150,7 +150,8 @@ class Costs: "preliminaries": preliminaries_cost, "material": base_material_cost, "profit": profit_cost, - "labour_hours": labour_hours + "labour_hours": labour_hours, + "labour_cost": labour_cost } def loft_insulation(self, floor_area, material): @@ -187,7 +188,8 @@ class Costs: "preliminaries": preliminaries_cost, "material": base_material_cost, "profit": profit_cost, - "labour_hours": labour_hours + "labour_hours": labour_hours, + "labour_cost": labour_cost } def internal_wall_insulation(self, wall_area, material, non_insulation_materials): @@ -280,7 +282,8 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -297,45 +300,6 @@ class Costs: :return: """ - # material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0, - # 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - # 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0, - # 'total_cost': 13.46, 'link': 'SPONs', - # 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. We use - # the ' - # 'same values as in Crown loft roll 44, since it is also an insulation roll'} - # - # non_insulation_materials = [ - # {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - # 'therefore there is no need for a skip'}, - # {'type': 'suspended_floor_demolition', - # 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, - # 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, - # 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - # 'therefore we need just labour rates'}] - - # insulation_floor_area = self.property.floor_area / self.property.number_of_floors - demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"] @@ -394,7 +358,8 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -428,78 +393,13 @@ class Costs: 7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments. - :param material: - :param non_insulation_materials: + + :param insulation_floor_area: Area of the floor to be insulated + :param material: Selected insulation material + :param non_insulation_materials: Non-insulation materials required for the job :return: """ - # material = { - # 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - # 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - # 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, - # 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 - # } - # - # non_insulation_materials = [ - # {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - # 'therefore there is no need for a skip'}, - # {'type': 'solid_floor_preparation', - # 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - # 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', - # 'description': 'Clean out crack to ' - # 'form a 20mm×20mm ' - # 'groove and fill with ' - # 'cement: mortar mixed ' - # 'with bonding agent', - # 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, - # 'material_cost': 6.91, - # 'labour_cost': 18.99, - # 'labour_hours_per_unit': 0.61, - # 'plant_cost': 0.16, - # 'total_cost': 26.06, 'link': 0, - # 'Notes': 'This step is the ' - # 'assessment and repair of ' - # 'any damage to the concrete ' - # 'floor such as filling ' - # 'cracks or levelling uneven ' - # 'areas'}, - # {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'solid_floor_redecoration', - # 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, - # 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', - # 'Notes': 'This is the screed layer, placed on top of the insulation'}, - # {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - # 'therefore we need just labour rates'}, - # {'type': 'solid_floor_redecoration', - # 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - # 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} - # ] - - # insulation_floor_area = self.property.floor_area / self.property.number_of_floors - demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"] preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"] @@ -564,7 +464,8 @@ class Costs: "material": materials_cost, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def external_wall_insulation(self, wall_area, material, non_insulation_materials): @@ -632,73 +533,6 @@ class Costs: else: raise ValueError("Unsupported property type - haven't handled flats") - # non_insulation_materials = [x for x in df if x["type"] != "external_wall_insulation"] - # insulation_materials = [x for x in df if x["type"] == "external_wall_insulation"] - # material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - # 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, - # 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, - # 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} - # non_insulation_materials = [ - # {'type': 'ewi_wall_demolition', - # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' - # 'hammer; plaster to walls.', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, - # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, - # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', - # 'description': 'Stud walls: Remove wall linings ' - # 'including battening behind; ' - # 'plasterboard and skim', - # 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, - # 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, - # 'plant_cost': 1.25, 'total_cost': 7.48, - # 'link': 'SPONs', 'Notes': 0}, - # {'type': 'ewi_wall_demolition', - # 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' - # 'behind; wood lath and plaster', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, - # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, - # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', - # 'description': 'Clean and prepare surfaces, ' - # 'one coat Keim dilution, ' - # 'one coat primer and two coats ' - # 'of Keim Ecosil paint; Brick or ' - # 'block walls; over 300 mm girth', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 7.3, - # 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, - # 'plant_cost': 0, 'total_cost': 12.92, - # 'link': 'SPONs', - # 'Notes': 'This work covers the preparation and ' - # 'priming of the wall before insulating'}, - # {'type': 'ewi_wall_redecoration', - # 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' - # 'separately) with horizontal PVC intermediate track and vertical ' - # 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' - # 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' - # 'Topcoat Render (white)', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, - # 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, - # 'total_cost': 69.94, 'link': 'SPONs', - # 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' - # 'cost of 99.17 per meter square. This includes the cost of insulation. ' - # 'To get the costing for just the works and not the insulation, ' - # 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' - # 'as an example, which costs £29.23 per meter square, giving us the cost ' - # 'of the remaining works without insulation. This material gives us a ' - # 'cost for basecoat, mesh application and a render finish'}] - demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] @@ -759,5 +593,6 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 8e2d76ce..1ba601a8 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -30,7 +30,7 @@ class TestCosts: assert cwi_results == {'total': 1027.0280465530302, 'subtotal': 855.8567054608585, 'vat': 171.1713410921717, 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574, 'profit': 95.09518949565093, - 'labour_hours': 6.234177828761786} + 'labour_hours': 6.234177828761786, 'labour_cost': 94.95132385344874} def test_loft_insulation(self): mock_property = Mock() @@ -56,7 +56,8 @@ class TestCosts: assert loft_results == {'total': 414.8496486, 'subtotal': 345.70804050000004, 'vat': 69.14160810000001, 'contingency': 25.608003000000004, 'preliminaries': 25.608003000000004, - 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685} + 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685, + 'labour_cost': 57.7808} def test_internal_wall_insulation(self): mock_property = Mock() @@ -173,4 +174,238 @@ class TestCosts: assert iwi_results == {'total': 6421.5484411659245, 'subtotal': 5351.29036763827, 'vat': 1070.258073527654, 'contingency': 573.3525393898148, 'preliminaries': 382.2350262598765, 'material': 1747.488000615996, 'profit': 573.3525393898148, - 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405} + 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405, + 'labour_cost': 1927.1602026551818} + + def test_suspended_floor_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + + sus_floor_material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', + 'depth': 140.0, + 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, + 'plant_cost': 0, + 'total_cost': 13.46, 'link': 'SPONs', + 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. ' + 'We use the ' + 'same values as in Crown loft roll 44, since it is also an insulation roll'} + + sus_floor_non_insulation_materials = [ + {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'suspended_floor_demolition', + 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, + 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, + 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}] + + sus_floor_results = costs.suspended_floor_insulation( + insulation_floor_area=33.5, + material=sus_floor_material, + non_insulation_materials=sus_floor_non_insulation_materials + ) + + assert sus_floor_results == { + 'total': 3003.366924, 'subtotal': 2502.80577, 'vat': 500.561154, + 'contingency': 185.39302, 'preliminaries': 185.39302, 'material': 483.405, + 'profit': 278.08952999999997, 'labour_hours': 54.940000000000005, + 'labour_days': 2.289166666666667, 'labour_cost': 1370.5252 + } + + def test_solid_floor_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + sol_floor_material = { + 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + } + + sol_floor_non_insulation_materials = [ + {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'solid_floor_preparation', + 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + 'description': 'Clean out crack to ' + 'form a 20mm×20mm ' + 'groove and fill with ' + 'cement: mortar mixed ' + 'with bonding agent', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, + 'material_cost': 6.91, + 'labour_cost': 18.99, + 'labour_hours_per_unit': 0.61, + 'plant_cost': 0.16, + 'total_cost': 26.06, 'link': 0, + 'Notes': 'This step is the ' + 'assessment and repair of ' + 'any damage to the concrete ' + 'floor such as filling ' + 'cracks or levelling uneven ' + 'areas'}, + {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'solid_floor_redecoration', + 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + 'Notes': 'This is the screed layer, placed on top of the insulation'}, + {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}, + {'type': 'solid_floor_redecoration', + 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + ] + + sol_floor_results = costs.solid_floor_insulation( + insulation_floor_area=33.5, + material=sol_floor_material, + non_insulation_materials=sol_floor_non_insulation_materials + ) + + assert sol_floor_results == { + 'total': 3962.021952, 'subtotal': 3301.68496, 'vat': 660.336992, 'contingency': 353.75196, + 'preliminaries': 235.83464, 'material': 1006.3399999999999, 'profit': 353.75196, 'labour_hours': 57.285, + 'labour_days': 2.386875, 'labour_cost': 1346.6464 + } + + def test_external_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire", + "property-type": "House", + "built-form": 'End-Terrace' + } + + costs = Costs(mock_property) + + ewi_material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', + 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, + 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, + 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} + ewi_non_insulation_materials = [ + {'type': 'ewi_wall_demolition', + 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' + 'hammer; plaster to walls.', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, + 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, + 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', + 'description': 'Stud walls: Remove wall linings ' + 'including battening behind; ' + 'plasterboard and skim', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, + 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, + 'plant_cost': 1.25, 'total_cost': 7.48, + 'link': 'SPONs', 'Notes': 0}, + {'type': 'ewi_wall_demolition', + 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' + 'behind; wood lath and plaster', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, + 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, + 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', + 'description': 'Clean and prepare surfaces, ' + 'one coat Keim dilution, ' + 'one coat primer and two coats ' + 'of Keim Ecosil paint; Brick or ' + 'block walls; over 300 mm girth', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 7.3, + 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, + 'plant_cost': 0, 'total_cost': 12.92, + 'link': 'SPONs', + 'Notes': 'This work covers the preparation and ' + 'priming of the wall before insulating'}, + {'type': 'ewi_wall_redecoration', + 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' + 'separately) with horizontal PVC intermediate track and vertical ' + 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' + 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' + 'Topcoat Render (white)', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, + 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, + 'total_cost': 69.94, 'link': 'SPONs', + 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' + 'cost of 99.17 per meter square. This includes the cost of insulation. ' + 'To get the costing for just the works and not the insulation, ' + 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' + 'as an example, which costs £29.23 per meter square, giving us the cost ' + 'of the remaining works without insulation. This material gives us a ' + 'cost for basecoat, mesh application and a render finish'}] + + ewi_results = costs.external_wall_insulation( + wall_area=95.9104281347967, + material=ewi_material, + non_insulation_materials=ewi_non_insulation_materials + ) + + assert ewi_results == { + 'total': 13590.909723215433, 'subtotal': 11325.758102679527, 'vat': 2265.1516205359053, + 'contingency': 808.9827216199662, 'preliminaries': 1213.4740824299492, + 'material': 4020.565147410677, 'profit': 1213.4740824299492, + 'labour_hours': 187.02533486285358, 'labour_days': 5.8445417144641745, + 'labour_cost': 3921.5600094613983 + } From b8ae3450761f8efd4cba7c38aace55974ed6a7d8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 19:17:58 +0000 Subject: [PATCH 67/75] Created costs elt app --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 29 ++++++++- etl/costs/README.md | 35 +++++++++++ etl/costs/app.py | 98 ++++++++++++++++++++++++++++++ etl/costs/requirements.txt | 5 ++ 6 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 etl/costs/README.md create mode 100644 etl/costs/app.py create mode 100644 etl/costs/requirements.txt diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..ed9033de 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..3ab974fc 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 812c1ebb..e191c5ee 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -19,6 +19,20 @@ class MaterialType(enum.Enum): flat_roof_insulation = "flat_roof_insulation" room_roof_insulation = "room_roof_insulation" + iwi_wall_demolition = "iwi_wall_demolition" + iwi_vapour_barrier = "iwi_vapour_barrier" + iwi_redecoration = "iwi_redecoration" + suspended_floor_demolition = "suspended_floor_demolition" + suspended_floor_redecoration = "suspended_floor_redecoration" + suspended_floor_vapour_barrier = "suspended_floor_vapour_barrier" + solid_floor_demolition = "solid_floor_demolition" + solid_floor_preparation = "solid_floor_preparation" + solid_floor_vapour_barrier = "solid_floor_vapour_barrier" + solid_floor_redecoration = "solid_floor_redecoration" + ewi_wall_demolition = "ewi_wall_demolition" + ewi_wall_preparation = "ewi_wall_preparation" + ewi_wall_redecoration = "ewi_wall_redecoration" + class DepthUnit(enum.Enum): mm = "mm" @@ -27,6 +41,7 @@ class DepthUnit(enum.Enum): class CostUnit(enum.Enum): gbp_sq_meter = "gbp_sq_meter" gbp_per_unit = "gbp_per_unit" + gbp_per_m2 = "gbp_per_m2" class RValueUnit(enum.Enum): @@ -41,9 +56,11 @@ class Material(Base): __tablename__ = 'material' id = Column(Integer, primary_key=True, autoincrement=True) - type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x]), nullable=False) + type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x], create_constraint=False), + nullable=False) + description = Column(String, nullable=False) - depths = Column(String) # You may want to use a specific JSON type depending on the database + depth = Column(String) # You may want to use a specific JSON type depending on the database depth_unit = Column(Enum(DepthUnit, values_callable=lambda x: [e.value for e in x]), nullable=False) cost = Column(String) cost_unit = Column(Enum(CostUnit, values_callable=lambda x: [e.value for e in x]), nullable=False) @@ -57,3 +74,11 @@ class Material(Base): link = Column(String) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) is_active = Column(Boolean, nullable=False, default=True) + + prime_material_cost = Column(Float) + material_cost = Column(Float) + labour_cost = Column(Float) + labour_hours_per_unit = Column(Float) + plant_cost = Column(Float) + total_cost = Column(Float) + notes = Column(String) diff --git a/etl/costs/README.md b/etl/costs/README.md new file mode 100644 index 00000000..969a3173 --- /dev/null +++ b/etl/costs/README.md @@ -0,0 +1,35 @@ +### Costs ETL Application + +This is a simple application to push the materials costs data to the database. + +#### How to run + +Ensure you have a .env file in the base Model directory with the following variables + +``` +DB_HOST="Your db host" +DB_PORT="Your db port" +DB_USER="Your db user" +DB_PASSWORD="Your db password" +DB_NAME="Your db name" +``` + +Make sure your python path environment variable pouints to the base Model directory. To set the +`PYTHONPATH` environment variable, run the following command from the base Model directory + +``` +export PYTHONPATH=`pwd` +``` + +From the base Model directory, install the requirements by running the following command + +``` +pip install -r etl/costs/requirements.txt +``` + +Then run the following command to run the application + +``` +python etl/costs/app.py +``` + diff --git a/etl/costs/app.py b/etl/costs/app.py new file mode 100644 index 00000000..0117a66e --- /dev/null +++ b/etl/costs/app.py @@ -0,0 +1,98 @@ +import os +import dotenv +import json +import pandas as pd +import numpy as np +from pathlib import Path +from sqlalchemy.orm import Session +from sqlalchemy import create_engine +from backend.app.db.models.materials import Material + +DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx" +# Environment file is at the same level as this file +ENV_FILE = Path(__file__).parent / "etl" / "costs" / ".env" +dotenv.load_dotenv(ENV_FILE) + +DB_USERNAME = os.getenv('DB_USERNAME') +DB_PASSWORD = os.getenv('DB_PASSWORD') +DB_HOST = os.getenv('DB_HOST') +DB_PORT = os.getenv('DB_PORT') +DB_NAME = os.getenv('DB_NAME') + + +def push_costs_to_db(engine, costs_df): + """ + Push costs DataFrame to the database. + + :param engine: The SQLAlchemy engine connected to your database. + :param costs_df: The DataFrame containing cost data. + """ + materials = [] + + for _, row in costs_df.iterrows(): + row_dict = row.to_dict() + + # Add other necessary transformations here + + # Create Material object and add it to the list + materials.append(Material(**row_dict)) + + # Use SQLAlchemy session for bulk insert + with Session(engine) as session: + session.bulk_save_objects(materials) + session.commit() + + +def app(): + """ + This application uploads the cost data to our database + + The most recent cost data can be found in OneDrive, in the + shared folder > 04. Product Development > Cost data > Hestia Materials.xlsx + + For the moment, the data is uploaded manually. In the future, we will automate this so the data can be + stored locally and then is uploaded from the local_data folder + :return: + """ + + connection_string = "postgresql+{drivername}://{username}:{password}@{server}:{port}/{dbname}" + db_string = connection_string.format( + drivername="psycopg2", # You'll need to use psycopg2 driver for PostgreSQL + username=DB_USERNAME, + password=DB_PASSWORD, + server=DB_HOST, + port=DB_PORT, + dbname=DB_NAME, + ) + + db_engine = create_engine(db_string, pool_size=5, max_overflow=5) + + cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0) + loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0) + iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0) + suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0) + solid_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="solid_floor_insulation", header=0) + ewi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="external_wall_insulation", header=0) + + # Form a single table to be uploaded + costs = pd.concat( + [ + cwi_costs, + loft_insulation_costs, + iwi_costs, + suspended_floor_costs, + solid_floor_costs, + ewi_costs, + ] + ) + + costs = costs.replace({np.nan: None}) + costs["depth"] = costs["depth"].fillna(0) + costs["depth"] = costs["depth"].astype(str) + + # Push the costs to the database + push_costs_to_db(db_engine, costs) + + +if __name__ == "__main__": + app() diff --git a/etl/costs/requirements.txt b/etl/costs/requirements.txt new file mode 100644 index 00000000..7d6afa9e --- /dev/null +++ b/etl/costs/requirements.txt @@ -0,0 +1,5 @@ +pandas==1.5.3 +sqlalchemy==2.0.19 +python-dotenv +psycopg2-binary +openpyxl \ No newline at end of file From 79ddd648274688d9ad9845bc2e4b5555937c6f91 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 19:44:44 +0000 Subject: [PATCH 68/75] deleted temp data from costs --- recommendations/Costs.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 8f54a79a..c1c9b42e 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,32 +1,5 @@ import numpy as np -# Example - delete me -from backend.Property import Property - -print("DELETE ME IN COSTS CLASS") -from epc_api.client import EpcClient - -epc_client = EpcClient(auth_token=AUTH_TOKEN) -p1 = Property( - postcode="NN1 5JY", - address1="2 South Terrace", - epc_client=epc_client, - id=0 -) - -p2 = Property( - postcode="PO12 4TY", - address1="25 Albert Street", - epc_client=epc_client, - id=0 -) - -p1.search_address_epc() -p2.search_address_epc() - -p1.set_basic_property_dimensions() -p2.set_basic_property_dimensions() - # This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, From 96553d0fc01b4ced14fc4ff45015da3e69a77de6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:00:35 +0000 Subject: [PATCH 69/75] Updating the floor recommendations class for new cost data --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 7 +- backend/app/plan/utils.py | 26 +++----- etl/costs/app.py | 10 ++- recommendations/FloorRecommendations.py | 88 ++++++++++++++++++------- recommendations/recommendation_utils.py | 12 ++-- 7 files changed, 93 insertions(+), 54 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index ed9033de..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 3ab974fc..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 83a57d07..f110b27a 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import ( - create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id + create_recommendation_scoring_data, prepare_materials, get_cleaned, insert_temp_recommendation_id ) from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 @@ -114,7 +114,7 @@ async def trigger_plan(body: PlanTriggerRequest): # the same data logger.info("Reading in materials and cleaned datasets") materials = get_materials(session) - materials_by_type = filter_materials(materials) + materials = prepare_materials(materials) cleaned = get_cleaned() logger.info("Getting components and epc recommendations") @@ -126,13 +126,14 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations_scoring_data = [] for p in input_properties: + # Property recommendations p.get_components(cleaned) property_recommendations = [] # Floor recommendations - floor_recommender = FloorRecommendations(property_instance=p, materials=materials_by_type["floor"]) + floor_recommender = FloorRecommendations(property_instance=p, materials=materials) floor_recommender.recommend() if floor_recommender.recommendations: diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index e2bf9d86..e3723b24 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -1,6 +1,5 @@ import pandas as pd from backend.Property import Property -from collections import defaultdict from utils.s3 import read_from_s3 from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value @@ -10,22 +9,13 @@ from backend.app.config import get_settings import msgpack -def filter_materials(materials): - materials_by_type = defaultdict(list) - - mapping = { - "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], - "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"], - "ventilation": ["mechanical_ventilation"], - "roof": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"] - } - - materials = [row2dict(material) for material in materials] - - for component, types in mapping.items(): - materials_by_type[component] = [part for part in materials if part["type"] in types] - - return dict(materials_by_type) +def prepare_materials(materials): + """ + This function will prepare the materials for recommendations + :param materials: list of materials, as retrieved from the database + :return: + """ + return [row2dict(material) for material in materials] def insert_temp_recommendation_id(property_recommendations): @@ -173,7 +163,7 @@ def create_recommendation_scoring_data( parts = recommendation["parts"] if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - + scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: diff --git a/etl/costs/app.py b/etl/costs/app.py index 0117a66e..1ecbbb5f 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -1,12 +1,12 @@ import os import dotenv -import json import pandas as pd import numpy as np from pathlib import Path from sqlalchemy.orm import Session from sqlalchemy import create_engine from backend.app.db.models.materials import Material +from recommendations.recommendation_utils import calculate_r_value_per_mm DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx" # Environment file is at the same level as this file @@ -90,6 +90,14 @@ def app(): costs["depth"] = costs["depth"].fillna(0) costs["depth"] = costs["depth"].astype(str) + costs["r_value_per_mm"] = costs.apply( + lambda row: calculate_r_value_per_mm(float(row["depth"]), row["thermal_conductivity"]), axis=1 + ) + costs["r_value_unit"] = "square_meter_kelvin_per_watt" + + for col in ["material_cost", "labour_cost", "labour_hours_per_unit", "plant_cost"]: + costs[col] = costs[col].fillna(0) + # Push the costs to the database push_costs_to_db(db_engine, costs) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 5b194e0d..641272a3 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -1,5 +1,8 @@ import math from typing import List + +import pandas as pd + from BaseUtility import Definitions from datatypes.enums import QuantityUnits from backend.Property import Property @@ -8,6 +11,7 @@ from recommendations.recommendation_utils import ( get_recommended_part, get_floor_u_value ) from recommendations.rdsap_tables import FLOOR_LEVEL_MAP +from recommendations.Costs import Costs class FloorRecommendations(Definitions): @@ -30,25 +34,41 @@ class FloorRecommendations(Definitions): materials: List, ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials - - self.suspended_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "suspended_floor_insulation" - ] - self.solid_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "solid_floor_insulation" + self.suspended_floor_insulation_materials = [ + part for part in materials if part["type"] == "suspended_floor_insulation" ] - self.exposed_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "exposed_floor_insulation" + self.suspended_floor_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "suspended_floor_demolition", "suspended_floor_redecoration", "suspended_floor_vapour_barrier" + ] ] + self.solid_floor_insulation_materials = [ + part for part in materials if part["type"] == "solid_floor_insulation" + ] + + self.solid_floor_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "solid_floor_demolition", "solid_floor_preparation", "solid_floor_vapour_barrier", + "solid_floor_redecoration" + ] + ] + + self.exposed_floor_insulation_materials = [ + part for part in materials if part["type"] == "exposed_floor_insulation" + ] + + # TODO: To be completed + self.exposed_floor_non_insulation_materials = [] + def recommend(self): u_value = self.property.floor["thermal_transmittance"] @@ -98,7 +118,11 @@ class FloorRecommendations(Definitions): if self.property.floor["is_suspended"]: # Given the U-value, we recommend underfloor insulation - self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts) + self.recommend_floor_insulation( + u_value=u_value, + insulation_materials=self.suspended_floor_insulation_materials, + non_insulation_materials=self.suspended_floor_non_insulation_materials + ) return if self.property.floor["is_solid"]: @@ -113,20 +137,23 @@ class FloorRecommendations(Definitions): raise NotImplementedError("Implement me!") @staticmethod - def _make_floor_description(part, depth): - return f"Install {depth}{part['depth_unit']} {part['description']} insulation" + def _make_floor_description(material): + return f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation" - def recommend_floor_insulation(self, u_value, parts): + def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials): """ This method is tasked with estimating the impact of performing suspended floor insulation :return: """ - lowest_selected_u_value = None - for part in parts: - for depth, cost_per_unit in zip(part["depths"], part["cost"]): + insulation_materials = pd.DataFrame(insulation_materials) - part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"]) + lowest_selected_u_value = None + for _, insulation_material_group in insulation_materials.groupby("description"): + + for _, material in insulation_material_group.iterrows(): + + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -137,26 +164,37 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - quantity = self.property.insulation_floor_area - estimated_cost = cost_per_unit * quantity + if material["type"] == "suspended_floor_insulation": + cost_result = self.costs.suspended_floor_insulation( + insulation_floor_area=self.property.insulation_floor_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + elif material["type"] == "solid_floor_insulation": + cost_result = self.costs.solid_floor_insulation( + insulation_floor_area=self.property.insulation_floor_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + else: + raise NotImplementedError("Implement me!") self.recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=depth, - quantity=quantity, + part=material.to_dict(), + quantity=self.property.insulation_floor_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ), ], "type": "floor_insulation", - "description": self._make_floor_description(part, depth), + "description": self._make_floor_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 217f313f..5bd77a2a 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -109,22 +109,21 @@ def update_lowest_selected_u_value(lowest_selected_u_value, new_u_value): return lowest_selected_u_value -def get_recommended_part(part, selected_depth, selected_total_cost, quantity, quantity_unit): +def get_recommended_part(part, cost_result, quantity, quantity_unit): """ Utility function to return a recommended part with the selected depth. :param part: part to be recommended - :param selected_depth: depth of the selected part - :param selected_total_cost: Total cost of the selected part + :param cost_result: Total cost of the selected part, as returned by the Cost class :param quantity: Quantity of the selected part :param quantity_unit: Unit of the quantity :return: """ recommended_part = deepcopy(part) - recommended_part["depths"] = [selected_depth] - recommended_part["estimated_cost"] = selected_total_cost recommended_part["quantity"] = quantity recommended_part["quantity_unit"] = quantity_unit + recommended_part.update(cost_result) + return recommended_part @@ -563,6 +562,9 @@ def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): :return: """ + if thermal_conductivity_w_mK is None: + return None + r_value_m2k_w = (thickness_mm / 1000) / thermal_conductivity_w_mK # Calculate R-value per mm From 80b4d2390c092af1a59ece21995d2cf67d9bb44a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:03:09 +0000 Subject: [PATCH 70/75] update floor recommendations for suspended floor --- backend/app/db/functions/materials_functions.py | 5 ++++- backend/app/plan/router.py | 5 ++--- backend/app/plan/utils.py | 10 ---------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/backend/app/db/functions/materials_functions.py b/backend/app/db/functions/materials_functions.py index f3c2f316..4d492946 100644 --- a/backend/app/db/functions/materials_functions.py +++ b/backend/app/db/functions/materials_functions.py @@ -1,4 +1,5 @@ from backend.app.db.models.materials import Material +from backend.app.db.utils import row2dict from functools import lru_cache @@ -16,4 +17,6 @@ def get_materials(session): materials = session.query(Material).filter(Material.is_active).all() - return materials if materials else [] + materials = materials if materials else [] + + return [row2dict(material) for material in materials] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index f110b27a..8d8ffe2d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import ( - create_recommendation_scoring_data, prepare_materials, get_cleaned, insert_temp_recommendation_id + create_recommendation_scoring_data, get_cleaned, insert_temp_recommendation_id ) from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 @@ -114,7 +114,6 @@ async def trigger_plan(body: PlanTriggerRequest): # the same data logger.info("Reading in materials and cleaned datasets") materials = get_materials(session) - materials = prepare_materials(materials) cleaned = get_cleaned() logger.info("Getting components and epc recommendations") @@ -141,7 +140,7 @@ async def trigger_plan(body: PlanTriggerRequest): # Wall recommendations - wall_recomender = WallRecommendations(property_instance=p, materials=materials_by_type["walls"]) + wall_recomender = WallRecommendations(property_instance=p, materials=materials) wall_recomender.recommend() if wall_recomender.recommendations: diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index e3723b24..b73ba874 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -4,20 +4,10 @@ from utils.s3 import read_from_s3 from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value -from backend.app.db.utils import row2dict from backend.app.config import get_settings import msgpack -def prepare_materials(materials): - """ - This function will prepare the materials for recommendations - :param materials: list of materials, as retrieved from the database - :return: - """ - return [row2dict(material) for material in materials] - - def insert_temp_recommendation_id(property_recommendations): """ Creates a temporary recommendation id which is needed for From cb52c9f7a3b2fe443f1bf08532ba3291bc639bcc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:15:32 +0000 Subject: [PATCH 71/75] Updated wall recommended partially for new costs --- recommendations/WallRecommendations.py | 88 +++++++++++++++++++------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 12085840..4595ef22 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -1,6 +1,8 @@ import math from typing import List +import pandas as pd + from datatypes.enums import QuantityUnits from backend.Property import Property from BaseUtility import Definitions @@ -9,6 +11,7 @@ from recommendations.recommendation_utils import ( get_recommended_part, get_wall_u_value ) from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION +from recommendations.Costs import Costs from utils.logger import setup_logger logger = setup_logger() @@ -50,13 +53,36 @@ class WallRecommendations(Definitions): materials: List ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials + self.cavity_wall_insulation_materials = [ + part for part in materials if part["type"] == "cavity_wall_insulation" + ] + + self.internal_wall_insulation_materials = [ + part for part in materials if part["type"] == "internal_wall_insulation" + ] + + self.internal_wall_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration" + ] + ] + + self.external_wall_insulation_materials = [ + part for part in materials if part["type"] == "external_wall_insulation" + ] + + self.external_wall_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration" + ] + ] @property def ewi_valid(self): @@ -200,15 +226,15 @@ class WallRecommendations(Definitions): self.recommendations = recommendations - def _find_insulation(self, parts, u_value): + def _find_insulation(self, u_value, insulation_materials, non_insulation_materials): + lowest_selected_u_value = None recommendations = [] - for part in parts: + for _, insulation_material_group in insulation_materials.groupby("description"): - for depth, cost_per_unit in zip(part["depths"], part["cost"]): - - part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"]) + for _, material in insulation_material_group.iterrows(): + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -225,27 +251,40 @@ class WallRecommendations(Definitions): # We allow a small tolerance for error so we don't discount the recommendation entirely if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = cost_per_unit * self.property.insulation_wall_area + if material["type"] == "internal_wall_insulation": + cost_result = self.costs.internal_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + elif material["type"] == "external_wall_insulation": + cost_result = self.costs.external_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + else: + raise ValueError("Invalid material type") recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=depth, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "wall_insulation", - "description": "Install " + self._make_description(part, depth), + "description": "Install " + self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) @@ -258,27 +297,32 @@ class WallRecommendations(Definitions): :return: """ - ewi_parts = [ - part for part in self.materials if part["type"] == "external_wall_insulation" - ] if self.ewi_valid else [] - - iwi_parts = [part for part in self.materials if part["type"] == "internal_wall_insulation"] - # Recommend external and internal wall insulation separately # Since external and internal wall insulation are sufficiently different, # we separate the logic for for recommending them, therefore we don't # consider diminishing returns between the two - ewi_recommendations = self._find_insulation(ewi_parts, u_value) - iwi_recommendations = self._find_insulation(iwi_parts, u_value) + ewi_recommendations = [] + if self.ewi_valid: + ewi_recommendations = self._find_insulation( + u_value=u_value, + insulation_materials=pd.DataFrame(self.external_wall_insulation_materials), + non_insulation_materials=self.external_wall_non_insulation_materials + ) + + iwi_recommendations = self._find_insulation( + u_value=u_value, + insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials), + non_insulation_materials=self.internal_wall_non_insulation_materials + ) self.recommendations += ewi_recommendations + iwi_recommendations self.prune_diminishing_recommendations() @staticmethod - def _make_description(part, depth): - return f"{depth}{part['depth_unit']} {part['description']}" + def _make_description(material): + return f"{int(material['depth'])}{material['depth_unit']} {material['description']}" def prune_diminishing_recommendations(self): # For any recommendations, if we have at least 1 reommendation that does not exhibit diminishing returns From bf2e6c1ebc68771b17c4ffa2dbd2a43490fb36c2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 09:58:13 +0000 Subject: [PATCH 72/75] Updating recommendation classes with new cost data --- backend/app/plan/router.py | 4 +- backend/app/plan/utils.py | 2 +- recommendations/Costs.py | 4 +- recommendations/FireplaceRecommendations.py | 2 +- recommendations/FloorRecommendations.py | 6 +- recommendations/RoofRecommendations.py | 67 ++++++++++++------- recommendations/VentilationRecommendations.py | 2 +- recommendations/WallRecommendations.py | 21 +++--- 8 files changed, 67 insertions(+), 41 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8d8ffe2d..fffc604e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -147,7 +147,7 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations.append(wall_recomender.recommendations) # Roof recommendations - roof_recommender = RoofRecommendations(property_instance=p, materials=materials_by_type["roof"]) + roof_recommender = RoofRecommendations(property_instance=p, materials=materials) roof_recommender.recommend() if roof_recommender.recommendations: @@ -156,7 +156,7 @@ async def trigger_plan(body: PlanTriggerRequest): # Ventilation recommendations ventilation_recomender = VentilationRecommendations( property_instance=p, - materials=materials_by_type["ventilation"] + materials=[part for part in materials if part["type"] == "mechanical_ventilation"] ) ventilation_recomender.recommend() diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index b73ba874..20b5db5b 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -154,7 +154,7 @@ def create_recommendation_scoring_data( if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) + scoring_dict["roof_insulation_thickness_ENDING"] = str(int(parts[0]["depth"])) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: # Fill missing roof u-values - this fill is not based on recommended upgrades diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c1c9b42e..a96e1215 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -113,7 +113,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * wall_area + labour_hours = material["labour_hours_per_unit"] * wall_area return { "total": total_cost, @@ -151,7 +151,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * floor_area + labour_hours = material["labour_hours_per_unit"] * floor_area return { "total": total_cost, diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 3e82b9d1..9524c75a 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -43,6 +43,6 @@ class FireplaceRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, - "cost": estimated_cost, + "total": estimated_cost, } ] diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 641272a3..96b1356c 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -127,7 +127,11 @@ class FloorRecommendations(Definitions): if self.property.floor["is_solid"]: # Given the U-value, we recommend solid floor insulation options which are usually solid foam - self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts) + self.recommend_floor_insulation( + u_value=u_value, + insulation_materials=self.solid_floor_insulation_materials, + non_insulation_materials=self.solid_floor_non_insulation_materials + ) return if self.property.floor["is_to_unheated_space"] or self.property.floor["is_to_external_air"]: diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 4f96f629..1bee1e8e 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -1,4 +1,5 @@ import math +import pandas as pd from backend.Property import Property from typing import List from datatypes.enums import QuantityUnits @@ -6,6 +7,7 @@ from recommendations.recommendation_utils import ( get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric ) +from recommendations.Costs import Costs class RoofRecommendations: @@ -27,13 +29,17 @@ class RoofRecommendations: materials: List ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials + self.loft_insulation_materials = [ + part for part in materials if part["type"] == "loft_insulation" + ] + self.loft_non_insulation_materials = [] def recommend(self): @@ -58,7 +64,7 @@ class RoofRecommendations: # If we have a u-value already, need to implement this if u_value: if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - # The floor is already compliant + # The Roof is already compliant return if self.property.data["transaction-type"] == "new dwelling": @@ -66,6 +72,10 @@ class RoofRecommendations: raise NotImplementedError("Implement me") u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) + self.estimated_u_value = u_value + if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + # The Roof is already compliant + return if self.property.roof["is_pitched"] or self.property.roof["is_flat"]: self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof) @@ -78,18 +88,21 @@ class RoofRecommendations: raise NotImplementedError("Implement me") @staticmethod - def make_loft_insulation_description(material, depth): - return f"Install {depth}{material['depth_unit']} of {material['description']} in your loft" + def make_loft_insulation_description(material): + return f"Install {int(material['depth'])}{material['depth_unit']} of {material['description']} in your loft" @staticmethod def make_room_roof_insulation_description(material, depth): return f"Insulate your room roof with {depth}{material['depth_unit']} of {material['description']}" @staticmethod - def make_flat_roof_insulation_description(material, depth): - return f"Insulate the home's flat roof with {depth}{material['depth_unit']} of {material['description']}" + def make_flat_roof_insulation_description(material): + return (f"Insulate the home's flat roof " + f"with {int(material['depth'])}{material['depth_unit']} of {material['description']}") - def recommend_roof_insulation(self, u_value, insulation_thickness, roof): + def recommend_roof_insulation( + self, u_value, insulation_thickness, roof + ): """ This method will recommend which insulation materials to use @@ -120,28 +133,31 @@ class RoofRecommendations: # from the base layer if roof["is_pitched"]: - materials = [m for m in self.materials if m["type"] == "loft_insulation"] + insulation_materials = self.loft_insulation_materials + non_insulation_materials = self.loft_non_insulation_materials elif roof["is_flat"]: - materials = [m for m in self.materials if m["type"] == "flat_roof_insulation"] + raise ValueError("UPDATE ME") else: raise ValueError("Roof is not pitched or flat") - if not materials: + if not insulation_materials: raise ValueError("No roof insulation materials found") + insulation_materials = pd.DataFrame(insulation_materials) + lowest_selected_u_value = None recommendations = [] - for material in materials: + for _, insulation_material_group in insulation_materials.groupby("description"): - for depth, cost_per_unit in zip(material["depths"], material["cost"]): + for _, material in insulation_material_group.iterrows(): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the # loft is already partially insulated. # Note: This requirement is only for loft insulation - if ((depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: + if ((material["depth"] + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: continue - part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -161,23 +177,26 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - # TODO: We should use the floor area divided by the number of floors to get the area of the roof - estimated_cost = cost_per_unit * self.property.floor_area - - if roof["is_pitched"]: - description = self.make_loft_insulation_description(material, depth) + if material["type"] == "loft_insulation": + cost_result = self.costs.loft_insulation( + floor_area=self.property.insulation_floor_area, + material=material + ) + description = self.make_loft_insulation_description(material) + elif material["type"] == "flat_roof_insulation": + description = self.make_flat_roof_insulation_description(material) + raise ValueError("COMPLETE ME") else: - description = self.make_flat_roof_insulation_description(material, depth) + raise ValueError("Invalid material type") recommendations.append( { "parts": [ get_recommended_part( - part=material, - selected_depth=depth, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "roof_insulation", @@ -185,7 +204,7 @@ class RoofRecommendations: "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index a639905b..a0b188f7 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -65,6 +65,6 @@ class VentilationRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, - "cost": estimated_cost, + "total": estimated_cost, } ] diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 4595ef22..acc74ead 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -180,7 +180,7 @@ class WallRecommendations(Definitions): filled cavity wall """ - cavity_wall_fills = [m for m in self.materials if m["type"] == "cavity_wall_insulation"] + insulation_materials = pd.DataFrame(self.cavity_wall_insulation_materials) cavity_width = 75 if insulation_thickness == "below average": cavity_width = cavity_width * (1 - PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION) @@ -188,8 +188,9 @@ class WallRecommendations(Definitions): # Test the different fill options lowest_selected_u_value = None recommendations = [] - for part in cavity_wall_fills: - part_u_value = r_value_per_mm_to_u_value(cavity_width, part["r_value_per_mm"]) + for _, material in insulation_materials.iterrows(): + + part_u_value = r_value_per_mm_to_u_value(cavity_width, material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -202,25 +203,27 @@ class WallRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = part["cost"] * self.property.insulation_wall_area + cost_result = self.costs.cavity_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + ) recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=None, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "wall_insulation", - "description": f"Fill cavity with {part['description']}", + "description": f"Fill cavity with {material['description']}", "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) From 02b76cd9fe6e24a30808352e6642799f11bbaacd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 11:01:44 +0000 Subject: [PATCH 73/75] Updating recommendation engine for new cost data --- backend/app/db/functions/recommendations_functions.py | 6 +++--- backend/app/plan/router.py | 1 - recommendations/VentilationRecommendations.py | 2 +- recommendations/optimiser/optimiser_functions.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 0bdf69ce..5d468db9 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -75,7 +75,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "property_id": property_id, "type": rec["type"], "description": rec["description"], - "estimated_cost": rec["cost"], + "estimated_cost": rec["total"], "default": rec["default"], "starting_u_value": rec.get("starting_u_value"), "new_u_value": rec.get("new_u_value"), @@ -102,10 +102,10 @@ def upload_recommendations(session: Session, recommendations_to_upload, property { "recommendation_id": recommendation_id, "material_id": part["id"], - "depth": part["depths"][0] if part["depths"] else None, + "depth": int(part["depth"]) if part["depth"] else None, "quantity": part["quantity"], "quantity_unit": part["quantity_unit"], - "estimated_cost": part["estimated_cost"], + "estimated_cost": part["total"], } for rec, recommendation_id in zip(recommendations_to_upload, uploaded_recommendation_ids) for part in rec["parts"] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fffc604e..37f673b4 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -318,7 +318,6 @@ async def trigger_plan(body: PlanTriggerRequest): batch_properties = input_properties[i:i + BATCH_SIZE] for p in batch_properties: - # Your existing operations property_details_epc = p.get_property_details_epc( portfolio_id=body.portfolio_id, rating_lookup=rating_lookup diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index a0b188f7..c330104f 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -52,7 +52,7 @@ class VentilationRecommendations(Definitions): estimated_cost = n_units * part[0]["cost"] - part[0]["estimated_cost"] = estimated_cost + part[0]["total"] = estimated_cost part[0]["quantity"] = n_units part[0]["quantity_unit"] = "part" diff --git a/recommendations/optimiser/optimiser_functions.py b/recommendations/optimiser/optimiser_functions.py index 869880cf..03aa38bd 100644 --- a/recommendations/optimiser/optimiser_functions.py +++ b/recommendations/optimiser/optimiser_functions.py @@ -22,7 +22,7 @@ def prepare_input_measures(property_recommendations, goal): [ { "id": rec["recommendation_id"], - "cost": rec["cost"], + "cost": rec["total"], "gain": rec[goal_key], "type": rec["type"] } From d942eacc557a032b84c7e9b67c4c55a80028a627 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 11:54:46 +0000 Subject: [PATCH 74/75] finished updating recommendations for now to build portfolios --- backend/app/db/functions/portfolio_functions.py | 2 ++ backend/app/db/functions/recommendations_functions.py | 3 ++- backend/app/plan/router.py | 2 +- recommendations/FireplaceRecommendations.py | 2 ++ recommendations/VentilationRecommendations.py | 4 +++- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/app/db/functions/portfolio_functions.py b/backend/app/db/functions/portfolio_functions.py index 37b6bf37..08e15a32 100644 --- a/backend/app/db/functions/portfolio_functions.py +++ b/backend/app/db/functions/portfolio_functions.py @@ -8,6 +8,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int): aggregates = ( session.query( func.sum(Recommendation.estimated_cost).label("cost"), + func.sum(Recommendation.total_work_hours).label("total_work_hours"), # For future usage we will aggregate multiple fields in this step # func.sum(Recommendation.heat_demand).label("total_heat_demand"), # func.sum(Recommendation.energy_savings).label("total_energy_savings") @@ -20,6 +21,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int): aggregates_dict = { "cost": aggregates.cost or 0, + "total_work_hours": aggregates.total_work_hours or 0, # "total_heat_demand": aggregates.total_heat_demand or 0, # "total_energy_savings": aggregates.total_energy_savings or 0 } diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 5d468db9..34c4ef96 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -79,7 +79,8 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "default": rec["default"], "starting_u_value": rec.get("starting_u_value"), "new_u_value": rec.get("new_u_value"), - "sap_points": rec["sap_points"] + "sap_points": rec["sap_points"], + "total_work_hours": rec["labour_hours"], } for rec in recommendations_to_upload ] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 37f673b4..a20369cc 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -367,7 +367,7 @@ async def trigger_plan(body: PlanTriggerRequest): # the portfolion level impact aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id) - # Commit all changes at once + # Commit final changes session.commit() except IntegrityError: logger.error("Database integrity error occurred", exc_info=True) diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 9524c75a..30ab1ad2 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -44,5 +44,7 @@ class FireplaceRecommendations(Definitions): "new_u_value": None, "sap_points": None, "total": estimated_cost, + # Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal + "labour_hours": 6 * number_open_fireplaces } ] diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index c330104f..419029a3 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -61,10 +61,12 @@ class VentilationRecommendations(Definitions): { "parts": part, "type": part[0]["type"], - "description": "Install %s" % part[0]["description"], + "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, "sap_points": None, "total": estimated_cost, + # We use a very simple and rough estimate of 4 hours per unit + "labour_hours": 4 * n_units } ] From 52bf5cdf1168066c29539a5bf5291e7ce0a1383e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 27 Nov 2023 14:53:41 +0000 Subject: [PATCH 75/75] Added infrastructure for heat and carbon models --- infrastructure/terraform/main.tf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index c7aacc4d..d545cdf8 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -129,6 +129,18 @@ module "retrofit_sap_data" { allowed_origins = var.allowed_origins } +module "retrofit_carbon_predictions" { + source = "./modules/s3" + bucketname = "retrofit-carbon-predictions-${var.stage}" + allowed_origins = var.allowed_origins +} + +module "retrofit_heat_predictions" { + source = "./modules/s3" + bucketname = "retrofit-heat-predictions-${var.stage}" + allowed_origins = var.allowed_origins +} + # Set up the route53 record for the API module "route53" { @@ -160,3 +172,13 @@ module "eco_spreadsheet_ecr" { ecr_name = "eco-spreadsheet-${var.stage}" source = "./modules/ecr" } + +module "lambda_carbon_prediction_ecr" { + ecr_name = "lambda-carbon-prediction-${var.stage}" + source = "./modules/ecr" +} + +module "lambda_heat_prediction_ecr" { + ecr_name = "lambda-heat-prediction-${var.stage}" + source = "./modules/ecr" +} \ No newline at end of file