diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py index 93567920..c27d05dc 100644 --- a/backend/tests/test_sap_model_prep.py +++ b/backend/tests/test_sap_model_prep.py @@ -15,7 +15,6 @@ import pickle with open("sap_change_dataset.pickle", "rb") as f: sap_change_dataset = pickle.load(f) -sap_change_dataset.columns.tolist() z = sap_change_dataset[(sap_change_dataset["NUMBER_OPEN_FIREPLACES_ENDING"] > 0) & ( sap_change_dataset["NUMBER_OPEN_FIREPLACES_STARTING"] > 0)].head(2).tail(1) z["UPRN"] @@ -24,81 +23,82 @@ z["SAP_ENDING"] z["NUMBER_OPEN_FIREPLACES_STARTING"] z["NUMBER_OPEN_FIREPLACES_ENDING"] - # 10002083298 # m -# search_from = sap_change_dataset[ -# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) -# ] -# search_from = search_from[ -# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & -# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & -# (search_from["MECHANICAL_VENTILATION_ENDING"] != search_from["MECHANICAL_VENTILATION_STARTING"]) & -# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & -# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) -# ] -# -# # Find a record where the only difference is cavity wall getting filled -# ending_cols = [c for c in search_from.columns if "_ENDING" in c] -# -# ignore = [ -# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", -# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -# ] -# -# ending_cols = [c for c in ending_cols if c not in ignore] -# -# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): -# -# same = True -# starting_cols = [] -# for c in ending_cols: -# -# starting_col = c.replace("_ENDING", "") -# if starting_col not in search_from.columns: -# starting_col = c.replace("_ENDING", "_STARTING") -# if starting_col not in search_from.columns: -# raise Exception("something went wrong") -# -# starting_cols.append(starting_col) -# -# # We want them to be different -# if c == "MECHANICAL_VENTILATION_ENDING": -# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): -# same = False -# break -# else: -# continue -# -# # We now check if the starting and ending values are the same -# if row[c] != row[starting_col]: -# same = False -# break -# -# if same: -# raise Exception("We found one!") -# -# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] -# -# import pandas as pd -# -# start = row[["SAP_STARTING"] + starting_cols] -# start.index = [c.replace("_STARTING", "") for c in start.index] -# end = row[["SAP_ENDING"] + ending_cols] -# end.index = [c.replace("_ENDING", "") for c in end.index] -# start["type"] = "starting" -# end["type"] = "ending" -# -# compare = pd.concat([start, end], axis=1) -# -# ending_lmk = "96b34dbe6cebdd7e648151e070047c8cd605c539851fc5a37e325903440081ab" -# starting_lmk = "dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247" -# -# client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "45 Shepperson Road", "postcode": "S6 4FG"}) -# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] +search_from = sap_change_dataset[ + (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) +] +search_from = search_from[ + (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & + (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) & + (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & + (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & + (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) & + (search_from["NUMBER_OPEN_FIREPLACES_STARTING"] > 0) & + (search_from["NUMBER_OPEN_FIREPLACES_ENDING"] == 0) + ] + +# Find a record where the only difference is cavity wall getting filled +ending_cols = [c for c in search_from.columns if "_ENDING" in c] + +ignore = [ + "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", + "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" +] + +ending_cols = [c for c in ending_cols if c not in ignore] + +for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): + + same = True + starting_cols = [] + for c in ending_cols: + + starting_col = c.replace("_ENDING", "") + if starting_col not in search_from.columns: + starting_col = c.replace("_ENDING", "_STARTING") + if starting_col not in search_from.columns: + raise Exception("something went wrong") + + starting_cols.append(starting_col) + + # We want them to be different + if c == "NUMBER_OPEN_FIREPLACES_ENDING": + if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): + same = False + break + else: + continue + + # We now check if the starting and ending values are the same + if row[c] != row[starting_col]: + same = False + break + + if same: + raise Exception("We found one!") + + fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] + + import pandas as pd + + start = row[["SAP_STARTING"] + starting_cols] + start.index = [c.replace("_STARTING", "") for c in start.index] + end = row[["SAP_ENDING"] + ending_cols] + end.index = [c.replace("_ENDING", "") for c in end.index] + start["type"] = "starting" + end["type"] = "ending" + + compare = pd.concat([start, end], axis=1) + +ending_lmk = "bab3983fa167717b8bb4a36ef395046d53937f9b880a45bcc751270d72e5de45" +starting_lmk = "736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef" + +client = EpcClient(auth_token=EPC_AUTH_TOKEN) +result = client.domestic.search(params={"address": "9 Glebe Road, Asfordby Hill", "postcode": "LE14 3QT"}) +starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] +ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] # with open( @@ -718,3 +718,189 @@ class TestSapModelPrep: continue assert test_record3[c].values[0] == row3[c] + + def test_fireplaces(self, cleaned, cleaning_data): + + starting_epc4 = { + 'low-energy-fixed-light-count': '', 'address': '9 Glebe Road, Asfordby Hill', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.4', 'heating-cost-potential': '501', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '70', + 'construction-age-band': 'England and Wales: 1930-1949', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', + 'environment-impact-potential': '76', 'glazed-type': 'double glazing, unknown install date', + 'heating-cost-current': '723', 'address3': '', + 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', + 'property-type': 'House', 'local-authority-label': 'Melton', + 'fixed-lighting-outlets-count': '14', 'energy-tariff': 'dual', + 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', + 'county': 'Leicestershire', 'postcode': 'LE14 3QT', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000909', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '5', + 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '153', + 'local-authority': 'E07000133', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '1', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', + 'inspection-date': '2022-06-27', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '46', + 'address1': '9 Glebe Road', 'heat-loss-corridor': '', 'flat-storey-count': '', + 'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Good', + 'total-floor-area': '87.0', 'building-reference-number': '10002396876', + 'environment-impact-current': '60', 'co2-emissions-current': '4.0', + 'roof-description': 'Pitched, 200 mm loft insulation', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Asfordby Hill', 'hot-water-env-eff': 'Good', + 'posttown': 'MELTON MOWBRAY', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Average', + 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 29% of fixed outlets', 'roof-env-eff': 'Good', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '15.0', 'lighting-cost-potential': '79', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', + 'lodgement-datetime': '2022-06-27 15:28:18', 'flat-top-storey': '', + 'current-energy-rating': 'D', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030539619', + 'current-energy-efficiency': '66', 'energy-consumption-current': '256', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '135', + 'lodgement-date': '2022-06-27', 'extension-count': '1', 'mainheatc-env-eff': 'Average', + 'lmk-key': '736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef', + 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', + 'potential-energy-efficiency': '78', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '29', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + } + + row4 = { + 'UPRN': '100030539619', 'RDSAP_CHANGE': 7, 'HEAT_DEMAND_CHANGE': -41, 'CARBON_CHANGE': -0.5, + 'SAP_STARTING': 66, 'SAP_ENDING': 73, 'HEAT_DEMAND_STARTING': 256, 'HEAT_DEMAND_ENDING': 215, + 'CARBON_STARTING': 4.0, 'CARBON_ENDING': 3.5, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Semi-Detached', + 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0, + 'FIXED_LIGHTING_OUTLETS_COUNT': 14.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1930-1949', + 'TRANSACTION_TYPE_STARTING': 'eco assessment', 'MECHANICAL_VENTILATION_STARTING': 'natural', + 'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_STARTING': 'dual', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 15.0, + 'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, + 'LOW_ENERGY_LIGHTING_STARTING': 29.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, + 'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 87.0, 'FLOOR_HEIGHT_STARTING': 2.4, + 'TRANSACTION_TYPE_ENDING': 'eco assessment', 'MECHANICAL_VENTILATION_ENDING': 'natural', + 'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)', + 'ENERGY_TARIFF_ENDING': 'dual', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 15.0, + 'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, + 'LOW_ENERGY_LIGHTING_ENDING': 29.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0, 'EXTENSION_COUNT_ENDING': 1.0, + 'TOTAL_FLOOR_AREA_ENDING': 87.0, 'FLOOR_HEIGHT_ENDING': 2.4, 'DAYS_TO_STARTING': 2887, + 'DAYS_TO_ENDING': 2960, 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, + 'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, + 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, + 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 1.7, 'is_park_home_ENDING': False, + 'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False, + 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.66, 'is_to_unheated_space': False, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, 'another_property_below': False, + 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.66, + 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.21, 'is_pitched': True, + 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'has_dwelling_above': False, 'roof_insulation_thickness': '200', 'roof_thermal_transmittance_ENDING': 0.21, + 'roof_insulation_thickness_ENDING': '200', 'heater_type': 'Unknown', 'system_type': 'from main system', + 'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', + 'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown', + 'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown', + 'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system', + 'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown', + 'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown', + 'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown', + 'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown', + 'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, + 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, + 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, + 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, + 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, + 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, + 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, + 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, + 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, + 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, + 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, + 'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False, + 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, + 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, + 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, + 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, + 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat', + 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', + 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, + 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown', + 'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown', + 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', + 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, + 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown', + 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'mains gas', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, + 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', + 'estimated_perimeter_STARTING': 37.54197650630557, 'estimated_perimeter_ENDING': 37.54197650630557 + } + + home4 = Property( + id=0, + postcode=starting_epc4["postcode"], + address1=starting_epc4["address1"], + epc_client=EpcClient(auth_token="notoken"), + data=starting_epc4 + ) + home4.get_components(cleaned) + + data_processor4 = DataProcessor(None, newdata=True) + data_processor4.insert_data(pd.DataFrame([home4.get_model_data()])) + + data_processor4.pre_process() + + starting_epc_data4 = data_processor4.get_component_features(suffix="_STARTING") + ending_epc_data4 = data_processor4.get_component_features(suffix="_ENDING") + fixed_data4 = data_processor4.get_fixed_features() + + ending_lodgement_date4 = '2022-09-08' + + ending_epc_data4["DAYS_TO_ENDING"] = data_processor4.calculate_days_to(ending_lodgement_date4) + + recommendation4 = { + "recommendation_id": 0, + "type": "sealing_open_fireplace" + } + + test_record4 = create_recommendation_scoring_data( + property=home4, + recommendation=recommendation4, + starting_epc_data=starting_epc_data4, + ending_epc_data=ending_epc_data4, + fixed_data=fixed_data4, + ) + test_record4 = pd.DataFrame([test_record4]) + + # Test the final cleaning: + test_record4 = DataProcessor.apply_averages_cleaning( + data_to_clean=test_record4, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + ).drop(columns=["LOCAL_AUTHORITY"]) + + test_record4 = DataProcessor.clean_missings_after_description_process( + test_record4, [ + c for c in test_record4.columns if + ("thermal_transmittance" in c) or ("insulation_thickness" in c) + ] + ) + + for c in test_record4.columns: + if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: + continue + + assert test_record4[c].values[0] == row4[c]