diff --git a/backend/tests/test_sap_model_prep.py b/backend/tests/test_sap_model_prep.py deleted file mode 100644 index 89c436ce..00000000 --- a/backend/tests/test_sap_model_prep.py +++ /dev/null @@ -1,1000 +0,0 @@ -from backend.Property import Property -from etl.epc.DataProcessor import DataProcessor -from backend.app.plan.utils import create_recommendation_scoring_data, get_cleaned -from etl.epc.settings import COLUMNS_TO_MERGE_ON -import pandas as pd -import pytest -import msgpack - -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 - - -# Handy code for selecting testing data -# import pickle -# -# with open("sap_dataset.pickle", "rb") as f: -# sap_change_dataset = pickle.load(f) -# -# search_from = sap_change_dataset[ -# (sap_change_dataset["walls_thermal_transmittance_ENDING"] == sap_change_dataset["walls_thermal_transmittance"]) & -# sap_change_dataset["is_to_unheated_space"] -# ] -# search_from = search_from[ -# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) & -# (search_from["floor_thermal_transmittance_ENDING"] != search_from["floor_thermal_transmittance"]) & -# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) & -# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) & -# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"]) -# ] -# -# # Find a record where the only difference is cavity wall getting filled -# ending_cols = [c for c in search_from.columns if "_ENDING" in c] -# -# ignore = [ -# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING", -# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING" -# ] -# -# ending_cols = [c for c in ending_cols if c not in ignore] -# -# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]): -# -# same = True -# starting_cols = [] -# for c in ending_cols: -# -# starting_col = c.replace("_ENDING", "") -# if starting_col not in search_from.columns: -# starting_col = c.replace("_ENDING", "_STARTING") -# if starting_col not in search_from.columns: -# raise Exception("something went wrong") -# -# starting_cols.append(starting_col) -# -# # We want them to be different -# if c == "floor_thermal_transmittance_ENDING": -# if (row[c] == row[starting_col]) | (row[starting_col] != "natural"): -# same = False -# break -# else: -# continue -# -# # We now check if the starting and ending values are the same -# if row[c] != row[starting_col]: -# same = False -# break -# -# if same: -# raise Exception("We found one!") -# -# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols] -# -# import pandas as pd -# -# start = row[["SAP_STARTING"] + starting_cols] -# start.index = [c.replace("_STARTING", "") for c in start.index] -# end = row[["SAP_ENDING"] + ending_cols] -# end.index = [c.replace("_ENDING", "") for c in end.index] -# start["type"] = "starting" -# end["type"] = "ending" -# -# compare = pd.concat([start, end], axis=1) -# -# ending_lmk = "1252008839062019090910572351658131" -# starting_lmk = "1252008819542014122308482236142128" -# -# client = EpcClient(auth_token=EPC_AUTH_TOKEN) -# result = client.domestic.search(params={"address": "Flat 14 Charles House, Freemens Way", "postcode": "CT14 9DL"}) -# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0] -# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0] - - -# with open( -# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaned.pickle", "rb" -# ) as f: -# cleaned = pickle.load(f) - -# with open( -# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaning_data.pickle", "rb" -# ) as f: -# cleaning_data = pickle.load(f) - -# TODO: Need to do floors, suspended and solid and to unheated space - - -class TestSapModelPrep: - - @pytest.fixture - def cleaning_data(self): - return read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - @pytest.fixture - def cleaned(self): - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - - cleaned = msgpack.unpackb(cleaned, raw=False) - return cleaned - - @pytest.fixture - def photo_supply_lookup(self): - photo_supply_lookup = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="solar_pv_supply/photo_supply_lookup.parquet", - ) - return photo_supply_lookup - - @pytest.fixture - def floor_area_decile_thresholds(self): - floor_area_decile_thresholds = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="solar_pv_supply/floor_area_decile_thresholds.parquet", - ) - return floor_area_decile_thresholds - - def test_fill_cavity_wall(self, cleaned, cleaning_data, photo_supply_lookup, floor_area_decile_thresholds): - """ - We ensure that the process that prepares the data in the engine code results in the same data as - the model is trained on - """ - - # This is an actual starting EPC - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '26, Vicarage Lane, Eaton', - 'uprn-source': 'Address Matched', 'floor-height': '2.39', 'heating-cost-potential': '942', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '97', - 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'D', - 'mainheat-energy-eff': 'Average', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Average', - 'environment-impact-potential': '53', - 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '1475', - 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', - 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Melton', - 'fixed-lighting-outlets-count': '', 'energy-tariff': 'Single', - 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96', 'county': 'Leicestershire', - 'postcode': 'NG32 1SP', 'solar-water-heating-flag': 'Y', 'constituency': 'E14000909', - 'co2-emissions-potential': '5.7', 'number-heated-rooms': '7', - 'floor-description': 'Suspended, no insulation (assumed)', - 'energy-consumption-potential': '177', 'local-authority': 'E07000133', 'built-form': 'Detached', - 'number-open-fireplaces': '1', 'windows-description': 'Fully double glazed', - 'glazed-area': 'Normal', 'inspection-date': '2016-09-22', 'mains-gas-flag': 'N', - 'co2-emiss-curr-per-floor-area': '87', 'address1': '26, Vicarage Lane', - 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', - 'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Very Poor', - 'total-floor-area': '116.0', 'building-reference-number': '4940047478', - 'environment-impact-current': '29', 'co2-emissions-current': '10.0', - 'roof-description': 'Pitched, limited insulation (assumed)', 'floor-energy-eff': 'NO DATA!', - 'number-habitable-rooms': '7', 'address2': 'Eaton', 'hot-water-env-eff': 'Good', - 'posttown': 'GRANTHAM', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'oil (not community)', - 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', - 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 31% of fixed outlets', - 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Poor', 'photo-supply': '', - 'lighting-cost-potential': '69', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', - 'main-heating-controls': '2106', 'lodgement-datetime': '2016-09-23 20:29:01', - 'flat-top-storey': '', 'current-energy-rating': 'F', - 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', 'walls-env-eff': 'Poor', - 'transaction-type': 'marketed sale', 'uprn': '100030534042', 'current-energy-efficiency': '34', - 'energy-consumption-current': '343', 'mainheat-description': 'Boiler and radiators, oil', - 'lighting-cost-current': '117', 'lodgement-date': '2016-09-23', 'extension-count': '2', - 'mainheatc-env-eff': 'Good', 'lmk-key': '1481856849902016092320290148762028', - 'wind-turbine-count': '0', 'tenure': 'owner-occupied', 'floor-level': 'NODATA!', - 'potential-energy-efficiency': '64', 'hot-water-energy-eff': 'Good', - 'low-energy-lighting': '31', - 'walls-description': 'Cavity wall, as built, no insulation (assumed)', - 'hotwater-description': 'From main system, plus solar' - } - - # This is the training data as we prepare it in the engine - # This is an actual record from the training data - row = { - 'UPRN': '100030534042', 'RDSAP_CHANGE': 12, 'HEAT_DEMAND_CHANGE': -72, - 'CARBON_CHANGE': -2.0999999999999996, 'SAP_STARTING': 34, 'SAP_ENDING': 46, 'HEAT_DEMAND_STARTING': 343, - 'HEAT_DEMAND_ENDING': 271, 'CARBON_STARTING': 10.0, 'CARBON_ENDING': 7.9, 'PROPERTY_TYPE': 'House', - 'BUILT_FORM': 'Detached', 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 7.0, - 'NUMBER_HEATED_ROOMS': 7.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 21.0, - 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1967-1975', 'TRANSACTION_TYPE_STARTING': 'marketed sale', - 'MECHANICAL_VENTILATION_STARTING': 'natural', - 'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)', - 'ENERGY_TARIFF_STARTING': 'Single', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'Y', - 'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002', - 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, 'LOW_ENERGY_LIGHTING_STARTING': 31.0, - 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, 'EXTENSION_COUNT_STARTING': 2.0, - 'TOTAL_FLOOR_AREA_STARTING': 116.0, 'FLOOR_HEIGHT_STARTING': 2.39, - 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural', - 'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)', - 'ENERGY_TARIFF_ENDING': 'Single', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'Y', 'PHOTO_SUPPLY_ENDING': 0.0, - 'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002', - 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, 'LOW_ENERGY_LIGHTING_ENDING': 31.0, - 'NUMBER_OPEN_FIREPLACES_ENDING': 1.0, 'EXTENSION_COUNT_ENDING': 2.0, 'TOTAL_FLOOR_AREA_ENDING': 116.0, - 'FLOOR_HEIGHT_ENDING': 2.41, 'DAYS_TO_STARTING': 784, 'DAYS_TO_ENDING': 867, - 'walls_thermal_transmittance': 1.5, 'is_cavity_wall': True, 'is_filled_cavity': False, - 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, - 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, - 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', - 'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.7, - 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average', - 'external_insulation_ENDING': False, 'internal_insulation_ENDING': False, - 'floor_thermal_transmittance': 0.52, 'is_to_unheated_space': False, 'is_to_external_air': False, - 'is_suspended': True, 'is_solid': False, 'another_property_below': False, - 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.52, - 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 1.5, 'is_pitched': True, - 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - 'has_dwelling_above': False, 'roof_insulation_thickness': 'below average', - 'roof_thermal_transmittance_ENDING': 1.5, 'roof_insulation_thickness_ENDING': 'below average', - 'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown', - 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown', - 'extra_features': 'plus solar', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', - 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'Unknown', - 'system_type_ENDING': 'from main system', 'thermostat_characteristics_ENDING': 'Unknown', - 'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown', - 'hotwater_tariff_type_ENDING': 'Unknown', 'extra_features_ENDING': 'plus solar', - 'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown', - 'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': True, - 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, - 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, - 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, - 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, - 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, - 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, - 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, - 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': True, - 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, - 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, - 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, - 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, - 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, - 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, - 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, - 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, - 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, - 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, - 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, - 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, - 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, - 'has_electric_ENDING': False, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False, - 'has_coal_ENDING': False, 'has_oil_ENDING': True, 'has_wood_pellets_ENDING': False, - 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, - 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, - 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, - 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat', - 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', - 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, - 'auxiliary_systems': 'Unknown', 'trvs': 'trvs', 'rate_control': 'Unknown', - 'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown', - 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', - 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, - 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'trvs', 'rate_control_ENDING': 'Unknown', - 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'oil', - 'main-fuel_tariff_type': 'Unknown', 'is_community': False, - 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', - 'fuel_type_ENDING': 'oil', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, - 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 30.531014675946444, 'estimated_perimeter_ENDING': 30.531014675946444, - 'HOT_WATER_ENERGY_EFF_STARTING': "Good", - "FLOOR_ENERGY_EFF_STARTING": "Unknown", - "WINDOWS_ENERGY_EFF_STARTING": "Good", - "WALLS_ENERGY_EFF_STARTING": "Poor", - "SHEATING_ENERGY_EFF_STARTING": "Unknown", - "ROOF_ENERGY_EFF_STARTING": "Very Poor", - "MAINHEAT_ENERGY_EFF_STARTING": "Average", - "MAINHEATC_ENERGY_EFF_STARTING": "Good", - "LIGHTING_ENERGY_EFF_STARTING": "Average", - "POTENTIAL_ENERGY_EFFICIENCY": 64, - "ENVIRONMENT_IMPACT_POTENTIAL": 53, - "ENERGY_CONSUMPTION_POTENTIAL": 177.0, - "CO2_EMISSIONS_POTENTIAL": 5.7, - "HOT_WATER_ENERGY_EFF_ENDING": "Good", - "FLOOR_ENERGY_EFF_ENDING": "Unknown", - "WINDOWS_ENERGY_EFF_ENDING": "Good", - "WALLS_ENERGY_EFF_ENDING": "Good", - "SHEATING_ENERGY_EFF_ENDING": "Unknown", - "ROOF_ENERGY_EFF_ENDING": "Very Poor", - "MAINHEAT_ENERGY_EFF_ENDING": "Average", - "MAINHEATC_ENERGY_EFF_ENDING": "Good", - "LIGHTING_ENERGY_EFF_ENDING": "Average", - } - - home = Property( - id=0, - postcode=starting_epc["postcode"], - address=starting_epc["address1"], - data=starting_epc - ) - home.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) - - data_processor = DataProcessor(None, newdata=True) - data_processor.insert_data(pd.DataFrame([home.get_model_data()])) - - data_processor.pre_process() - - starting_epc_data = data_processor.get_component_features(suffix="_STARTING") - ending_epc_data = data_processor.get_component_features(suffix="_ENDING") - fixed_data = data_processor.get_fixed_features() - - ending_lodgement_date = '2016-12-15' - - ending_epc_data["DAYS_TO_ENDING"] = data_processor.calculate_days_to(ending_lodgement_date) - - recommendation = { - "recommendation_id": 0, - "new_u_value": 0.7, - "type": "cavity_wall_insulation" - } - - test_record = create_recommendation_scoring_data( - property=home, - recommendation=recommendation, - starting_epc_data=starting_epc_data, - ending_epc_data=ending_epc_data, - fixed_data=fixed_data, - ) - test_record = pd.DataFrame([test_record]) - - # Test the final cleaning: - test_record = DataProcessor.apply_averages_cleaning( - data_to_clean=test_record, - cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] - ).drop(columns=["LOCAL_AUTHORITY"]) - - test_record = DataProcessor.clean_missings_after_description_process( - test_record, [ - c for c in test_record.columns if - ("thermal_transmittance" in c) or ("insulation_thickness" in c) - ] - ) - - # Test that the data has been set up correctly - - # Things to fix: - # [] Filled cavity should have an average insulation thickness in the cleaned data - - for c in test_record.columns: - if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: - continue - - if c == "FLOOR_HEIGHT_ENDING": - assert (row[c] - test_record[c].values[0]) <= 0.020001 - continue - - if c == "walls_insulation_thickness_ENDING": - assert row[c] == "average" - assert test_record[c].values[0] == "above average" - continue - - assert test_record[c].values[0] == row[c] - - def test_internal_wall_insulation(self, cleaned, cleaning_data, photo_supply_lookup, floor_area_decile_thresholds): - - starting_epc2 = { - 'low-energy-fixed-light-count': '2', 'address': 'FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY', - 'uprn-source': 'Energy Assessor', 'floor-height': '3.64', 'heating-cost-potential': '465', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '185', - 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C', - 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Poor', - 'environment-impact-potential': '51', 'glazed-type': 'double glazing installed during or after 2002', - 'heating-cost-current': '1223', 'address3': '3 WESTERN GATEWAY', - 'mainheatcont-description': 'Programmer and appliance thermostats', 'sheating-energy-eff': 'N/A', - 'property-type': 'Flat', 'local-authority-label': 'Newham', 'fixed-lighting-outlets-count': '12', - 'energy-tariff': 'off-peak 7 hour', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '342', - 'county': '', 'postcode': 'E16 1BD', 'solar-water-heating-flag': 'N', 'constituency': 'E14001032', - 'co2-emissions-potential': '3.6', 'number-heated-rooms': '2', 'floor-description': '(other premises below)', - 'energy-consumption-potential': '307', 'local-authority': 'E09000025', 'built-form': 'Mid-Terrace', - 'number-open-fireplaces': '0', 'windows-description': 'Partial double glazing', 'glazed-area': 'Normal', - 'inspection-date': '2020-10-14', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '66', - 'address1': 'FLAT 12', 'heat-loss-corridor': 'heated corridor', 'flat-storey-count': '', - 'constituency-label': 'West Ham', 'roof-energy-eff': 'N/A', 'total-floor-area': '70.0', - 'building-reference-number': '10000539740', 'environment-impact-current': '42', - 'co2-emissions-current': '4.6', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '2', 'address2': 'WAREHOUSE W', 'hot-water-env-eff': 'Poor', 'posttown': 'LONDON', - 'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Poor', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 17% of fixed outlets', 'roof-env-eff': 'N/A', - 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '67', - 'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '61', 'main-heating-controls': '', - 'lodgement-datetime': '2020-10-14 00:00:00', 'flat-top-storey': 'N', 'current-energy-rating': 'F', - 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'marketed sale', - 'uprn': '10012839482', 'current-energy-efficiency': '33', 'energy-consumption-current': '393', - 'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '110', - 'lodgement-date': '2020-10-14', 'extension-count': '0', 'mainheatc-env-eff': 'Good', - 'lmk-key': 'b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '01', 'potential-energy-efficiency': '71', - 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '17', - 'walls-description': 'Solid brick, as built, no insulation (assumed)', - 'hotwater-description': 'Electric immersion, standard tariff' - } - - row2 = { - 'UPRN': '10012839482', 'RDSAP_CHANGE': 8, 'HEAT_DEMAND_CHANGE': -59, - 'CARBON_CHANGE': -0.5999999999999996, 'SAP_STARTING': 33, 'SAP_ENDING': 41, 'HEAT_DEMAND_STARTING': 393, - 'HEAT_DEMAND_ENDING': 334, 'CARBON_STARTING': 4.6, 'CARBON_ENDING': 4.0, 'PROPERTY_TYPE': 'Flat', - 'BUILT_FORM': 'Mid-Terrace', 'CONSTITUENCY': 'E14001032', 'NUMBER_HABITABLE_ROOMS': 2.0, - 'NUMBER_HEATED_ROOMS': 2.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 12.0, - 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1996-2002', 'TRANSACTION_TYPE_STARTING': 'marketed sale', - 'MECHANICAL_VENTILATION_STARTING': 'natural', 'SECONDHEAT_DESCRIPTION_STARTING': 'None', - 'ENERGY_TARIFF_STARTING': 'off-peak 7 hour', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', - 'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002', - 'MULTI_GLAZE_PROPORTION_STARTING': 61.0, 'LOW_ENERGY_LIGHTING_STARTING': 17.0, - 'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, 'EXTENSION_COUNT_STARTING': 0.0, - 'TOTAL_FLOOR_AREA_STARTING': 70.0, 'FLOOR_HEIGHT_STARTING': 3.64, - 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural', - 'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'off-peak 7 hour', - 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0, - 'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002', - 'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0, - 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0, - 'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307, - 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, 'is_filled_cavity': False, - 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, - 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, - 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', - 'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.21, - 'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average', - 'external_insulation_ENDING': False, 'internal_insulation_ENDING': False, - 'floor_thermal_transmittance': 0.0, 'is_to_unheated_space': False, 'is_to_external_air': False, - 'is_suspended': False, 'is_solid': False, 'another_property_below': True, - 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.0, - 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.0, 'is_pitched': False, - 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - 'has_dwelling_above': True, 'roof_insulation_thickness': 'none', - 'roof_thermal_transmittance_ENDING': 0.0, 'roof_insulation_thickness_ENDING': 'none', - 'heater_type': 'electric immersion', 'system_type': 'Unknown', 'thermostat_characteristics': 'Unknown', - 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'standard tariff', - 'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', - 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'electric immersion', - 'system_type_ENDING': 'Unknown', 'thermostat_characteristics_ENDING': 'Unknown', - 'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown', - 'hotwater_tariff_type_ENDING': 'standard tariff', 'extra_features_ENDING': 'Unknown', - 'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown', - 'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': False, - 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, - 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, - 'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False, - 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, - 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, - 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, - 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': True, - 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, - 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, - 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, - 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': False, - 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, - 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, - 'has_boiler_ENDING': False, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': True, - 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, - 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, - 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, - 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, - 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, - 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, - 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, - 'has_electric_ENDING': True, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False, - 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, - 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, - 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, - 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, - 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'appliance thermostats', - 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', - 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, - 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown', - 'thermostatic_control_ENDING': 'appliance thermostats', 'charging_system_ENDING': 'Unknown', - 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', - 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, - 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown', - 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'electricity', - 'main-fuel_tariff_type': 'Unknown', 'is_community': False, - 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', - 'fuel_type_ENDING': 'electricity', 'main-fuel_tariff_type_ENDING': 'Unknown', - 'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False, - 'complex_fuel_type_ENDING': 'Unknown', 'estimated_perimeter_STARTING': 35.4964786985977, - 'estimated_perimeter_ENDING': 35.4964786985977, - 'HOT_WATER_ENERGY_EFF_STARTING': "Very Poor", - "FLOOR_ENERGY_EFF_STARTING": "Unknown", - "WINDOWS_ENERGY_EFF_STARTING": "Average", - "WALLS_ENERGY_EFF_STARTING": "Very Poor", - "SHEATING_ENERGY_EFF_STARTING": "Unknown", - "ROOF_ENERGY_EFF_STARTING": "Unknown", - "MAINHEAT_ENERGY_EFF_STARTING": "Very Poor", - "MAINHEATC_ENERGY_EFF_STARTING": "Good", - "LIGHTING_ENERGY_EFF_STARTING": "Poor", - "POTENTIAL_ENERGY_EFFICIENCY": 71, - "ENVIRONMENT_IMPACT_POTENTIAL": 51, - "ENERGY_CONSUMPTION_POTENTIAL": 307, - "CO2_EMISSIONS_POTENTIAL": 3.6, - 'HOT_WATER_ENERGY_EFF_ENDING': "Very Poor", - "FLOOR_ENERGY_EFF_ENDING": "Unknown", - "WINDOWS_ENERGY_EFF_ENDING": "Average", - "WALLS_ENERGY_EFF_ENDING": "Good", - "SHEATING_ENERGY_EFF_ENDING": "Unknown", - "ROOF_ENERGY_EFF_ENDING": "Unknown", - "MAINHEAT_ENERGY_EFF_ENDING": "Very Poor", - "MAINHEATC_ENERGY_EFF_ENDING": "Good", - "LIGHTING_ENERGY_EFF_ENDING": "Poor", - } - - home2 = Property( - id=0, - postcode=starting_epc2["postcode"], - address=starting_epc2["address1"], - data=starting_epc2 - ) - home2.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) - home2.set_number_lighting_outlets(None) - - data_processor2 = DataProcessor(None, newdata=True) - data_processor2.insert_data(pd.DataFrame([home2.get_model_data()])) - - data_processor2.pre_process() - - starting_epc_data2 = data_processor2.get_component_features(suffix="_STARTING") - ending_epc_data2 = data_processor2.get_component_features(suffix="_ENDING") - fixed_data2 = data_processor2.get_fixed_features() - - ending_lodgement_date2 = '2020-11-24' - - ending_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2) - - recommendation2 = { - "recommendation_id": 0, - "new_u_value": 0.21, - "type": "internal_wall_insulation" - } - - test_record2 = create_recommendation_scoring_data( - property=home2, - recommendation=recommendation2, - starting_epc_data=starting_epc_data2, - ending_epc_data=ending_epc_data2, - fixed_data=fixed_data2, - ) - test_record2 = pd.DataFrame([test_record2]) - - # Test the final cleaning: - test_record2 = DataProcessor.apply_averages_cleaning( - data_to_clean=test_record2, - cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] - ).drop(columns=["LOCAL_AUTHORITY"]) - - test_record2 = DataProcessor.clean_missings_after_description_process( - test_record2, [ - c for c in test_record2.columns if - ("thermal_transmittance" in c) or ("insulation_thickness" in c) - ] - ) - - for c in test_record2.columns: - if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: - continue - - if c == "FLOOR_HEIGHT_ENDING": - assert (row2[c] - test_record2[c].values[0]) <= 0.020001 - continue - - if c == "walls_insulation_thickness_ENDING": - assert row2[c] == "average" - assert test_record2[c].values[0] == "above average" - continue - - if c == "CONSTRUCTION_AGE_BAND": - # For this, we have different values in the original data - assert row2[c] == "England and Wales: 1996-2002" - assert test_record2[c].values[0] == "England and Wales: 1900-1929" - continue - - assert test_record2[c].values[0] == row2[c] - - def test_ventilation(self, cleaned, cleaning_data, photo_supply_lookup, floor_area_decile_thresholds): - - starting_epc3 = { - 'low-energy-fixed-light-count': '', 'address': '45 Shepperson Road', 'uprn-source': 'Energy Assessor', - 'floor-height': '1.87', 'heating-cost-potential': '645', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '69', 'construction-age-band': 'England and Wales: 1900-1929', - 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', - 'lighting-energy-eff': 'Average', 'environment-impact-potential': '75', - 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '1028', 'address3': '', - 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'Sheffield', 'fixed-lighting-outlets-count': '21', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '96', - 'county': '', 'postcode': 'S6 4FG', 'solar-water-heating-flag': 'N', 'constituency': 'E14000921', - 'co2-emissions-potential': '2.9', 'number-heated-rooms': '5', - 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '152', - 'local-authority': 'E08000019', 'built-form': 'Enclosed Mid-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-06-13', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '59', 'address1': '45 Shepperson Road', - 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Sheffield, Brightside and Hillsborough', 'roof-energy-eff': 'Very Poor', - 'total-floor-area': '107.0', 'building-reference-number': '10002892085', 'environment-impact-current': '46', - 'co2-emissions-current': '6.3', 'roof-description': 'Pitched, no insulation (assumed)', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', - 'posttown': 'SHEFFIELD', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'mains gas (not community)', - 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', - 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 43% of fixed outlets', - 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', - 'lighting-cost-potential': '83', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', - 'main-heating-controls': '', 'lodgement-datetime': '2023-05-27 12:15:21', 'flat-top-storey': '', - 'current-energy-rating': 'E', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', - 'transaction-type': 'marketed sale', 'uprn': '100051073214', 'current-energy-efficiency': '54', - 'energy-consumption-current': '335', 'mainheat-description': 'Boiler and radiators, mains gas', - 'lighting-cost-current': '131', 'lodgement-date': '2023-05-27', 'extension-count': '1', - 'mainheatc-env-eff': 'Average', - 'lmk-key': 'dc1a4da246562656132b8e36e0534cd90b09fa40fc584e25e644e2d9ab86a247', 'wind-turbine-count': '0', - 'tenure': 'Not defined - use in the case of a new dwelling for which the intended tenure in not known. It ' - 'is not to be used for an existing dwelling', - 'floor-level': '', 'potential-energy-efficiency': '80', 'hot-water-energy-eff': 'Good', - 'low-energy-lighting': '43', - 'walls-description': 'Sandstone or limestone, as built, no insulation (assumed)', - 'hotwater-description': 'From main system' - } - - row3 = { - 'UPRN': '100051073214', 'RDSAP_CHANGE': 2, 'HEAT_DEMAND_CHANGE': -22, 'CARBON_CHANGE': -0.39999999999999947, - 'SAP_STARTING': 54, 'SAP_ENDING': 56, 'HEAT_DEMAND_STARTING': 335, 'HEAT_DEMAND_ENDING': 313, - 'CARBON_STARTING': 6.3, 'CARBON_ENDING': 5.9, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Mid-Terrace', - 'CONSTITUENCY': 'E14000921', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0, - 'FIXED_LIGHTING_OUTLETS_COUNT': 21.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1900-1929', - 'TRANSACTION_TYPE_STARTING': 'marketed sale', 'MECHANICAL_VENTILATION_STARTING': 'natural', - 'SECONDHEAT_DESCRIPTION_STARTING': 'None', 'ENERGY_TARIFF_STARTING': 'Single', - 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 0.0, - 'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, - 'LOW_ENERGY_LIGHTING_STARTING': 43.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, - 'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 107.0, 'FLOOR_HEIGHT_STARTING': 1.87, - 'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'mechanical, extract only', - 'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'Single', - 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0, - 'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, - 'LOW_ENERGY_LIGHTING_ENDING': 43.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 1.0, - 'TOTAL_FLOOR_AREA_ENDING': 107.0, 'FLOOR_HEIGHT_ENDING': 1.87, 'DAYS_TO_STARTING': 3221, - 'DAYS_TO_ENDING': 2874, 'walls_thermal_transmittance': 2.0, 'is_cavity_wall': False, - 'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, - 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': True, - 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, - 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 2.0, 'is_park_home_ENDING': False, - 'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False, - 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.51, 'is_to_unheated_space': False, - 'is_to_external_air': False, 'is_suspended': True, 'is_solid': False, 'another_property_below': False, - 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.51, - 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 2.3, 'is_pitched': True, - 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - 'has_dwelling_above': False, 'roof_insulation_thickness': 'none', 'roof_thermal_transmittance_ENDING': 2.3, - 'roof_insulation_thickness_ENDING': 'none', 'heater_type': 'Unknown', 'system_type': 'from main system', - 'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', - 'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown', - 'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown', - 'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system', - 'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown', - 'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown', - 'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown', - 'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown', - 'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, - 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, - 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, - 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, - 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, - 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, - 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, - 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, - 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, - 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, - 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, - 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, - 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, - 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, - 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, - 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, - 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, - 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, - 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, - 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, - 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, - 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, - 'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False, - 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, - 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, - 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, - 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, - 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'Unknown', 'charging_system': 'Unknown', - 'switch_system': 'programmer', 'no_control': 'Unknown', 'dhw_control': 'Unknown', - 'community_heating': 'Unknown', 'multiple_room_thermostats': False, 'auxiliary_systems': 'bypass', - 'trvs': 'trvs', 'rate_control': 'Unknown', 'thermostatic_control_ENDING': 'Unknown', - 'charging_system_ENDING': 'Unknown', 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', - 'dhw_control_ENDING': 'Unknown', 'community_heating_ENDING': 'Unknown', - 'multiple_room_thermostats_ENDING': False, 'auxiliary_systems_ENDING': 'bypass', 'trvs_ENDING': 'trvs', - 'rate_control_ENDING': 'Unknown', 'glazing_type': 'double', 'glazing_type_ENDING': 'double', - 'fuel_type': 'mains gas', 'main-fuel_tariff_type': 'Unknown', 'is_community': False, - 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', - 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, - 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 30.06908711617298, 'estimated_perimeter_ENDING': 30.06908711617298, - 'HOT_WATER_ENERGY_EFF_STARTING': "Good", - "FLOOR_ENERGY_EFF_STARTING": "Unknown", - "WINDOWS_ENERGY_EFF_STARTING": "Average", - "WALLS_ENERGY_EFF_STARTING": "Very Poor", - "SHEATING_ENERGY_EFF_STARTING": "Unknown", - "ROOF_ENERGY_EFF_STARTING": "Very Poor", - "MAINHEAT_ENERGY_EFF_STARTING": "Good", - "MAINHEATC_ENERGY_EFF_STARTING": "Average", - "LIGHTING_ENERGY_EFF_STARTING": "Average", - "POTENTIAL_ENERGY_EFFICIENCY": 80, - "ENVIRONMENT_IMPACT_POTENTIAL": 75, - "ENERGY_CONSUMPTION_POTENTIAL": 152, - "CO2_EMISSIONS_POTENTIAL": 2.9, - 'HOT_WATER_ENERGY_EFF_ENDING': "Good", - "FLOOR_ENERGY_EFF_ENDING": "Unknown", - "WINDOWS_ENERGY_EFF_ENDING": "Average", - "WALLS_ENERGY_EFF_ENDING": "Very Poor", - "SHEATING_ENERGY_EFF_ENDING": "Unknown", - "ROOF_ENERGY_EFF_ENDING": "Very Poor", - "MAINHEAT_ENERGY_EFF_ENDING": "Good", - "MAINHEATC_ENERGY_EFF_ENDING": "Average", - "LIGHTING_ENERGY_EFF_ENDING": "Average", - } - - home3 = Property( - id=0, - postcode=starting_epc3["postcode"], - address=starting_epc3["address1"], - data=starting_epc3 - ) - home3.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) - home3.set_number_lighting_outlets(None) - - data_processor3 = DataProcessor(None, newdata=True) - data_processor3.insert_data(pd.DataFrame([home3.get_model_data()])) - - data_processor3.pre_process() - - starting_epc_data3 = data_processor3.get_component_features(suffix="_STARTING") - ending_epc_data3 = data_processor3.get_component_features(suffix="_ENDING") - fixed_data3 = data_processor3.get_fixed_features() - - ending_lodgement_date3 = '2022-06-14' - - ending_epc_data3["DAYS_TO_ENDING"] = data_processor3.calculate_days_to(ending_lodgement_date3) - - recommendation3 = { - "recommendation_id": 0, - "type": "mechanical_ventilation" - } - - test_record3 = create_recommendation_scoring_data( - property=home3, - recommendation=recommendation3, - starting_epc_data=starting_epc_data3, - ending_epc_data=ending_epc_data3, - fixed_data=fixed_data3, - ) - test_record3 = pd.DataFrame([test_record3]) - - # Test the final cleaning: - test_record3 = DataProcessor.apply_averages_cleaning( - data_to_clean=test_record3, - cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] - ).drop(columns=["LOCAL_AUTHORITY"]) - - test_record3 = DataProcessor.clean_missings_after_description_process( - test_record3, [ - c for c in test_record3.columns if - ("thermal_transmittance" in c) or ("insulation_thickness" in c) - ] - ) - - for c in test_record3.columns: - if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: - continue - - assert test_record3[c].values[0] == row3[c] - - def test_fireplaces(self, cleaned, cleaning_data, photo_supply_lookup, floor_area_decile_thresholds): - - starting_epc4 = { - 'low-energy-fixed-light-count': '', 'address': '9 Glebe Road, Asfordby Hill', - 'uprn-source': 'Energy Assessor', 'floor-height': '2.4', 'heating-cost-potential': '501', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '70', - 'construction-age-band': 'England and Wales: 1930-1949', 'potential-energy-rating': 'C', - 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', - 'environment-impact-potential': '76', 'glazed-type': 'double glazing, unknown install date', - 'heating-cost-current': '723', 'address3': '', - 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'Melton', - 'fixed-lighting-outlets-count': '14', 'energy-tariff': 'dual', - 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', - 'county': 'Leicestershire', 'postcode': 'LE14 3QT', 'solar-water-heating-flag': 'N', - 'constituency': 'E14000909', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '5', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '153', - 'local-authority': 'E07000133', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '1', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', - 'inspection-date': '2022-06-27', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '46', - 'address1': '9 Glebe Road', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Rutland and Melton', 'roof-energy-eff': 'Good', - 'total-floor-area': '87.0', 'building-reference-number': '10002396876', - 'environment-impact-current': '60', 'co2-emissions-current': '4.0', - 'roof-description': 'Pitched, 200 mm loft insulation', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '5', 'address2': 'Asfordby Hill', 'hot-water-env-eff': 'Good', - 'posttown': 'MELTON MOWBRAY', 'mainheatc-energy-eff': 'Average', - 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Average', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 29% of fixed outlets', 'roof-env-eff': 'Good', - 'walls-energy-eff': 'Very Poor', 'photo-supply': '15.0', 'lighting-cost-potential': '79', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2022-06-27 15:28:18', 'flat-top-storey': '', - 'current-energy-rating': 'D', - 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', - 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030539619', - 'current-energy-efficiency': '66', 'energy-consumption-current': '256', - 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '135', - 'lodgement-date': '2022-06-27', 'extension-count': '1', 'mainheatc-env-eff': 'Average', - 'lmk-key': '736b6f4803a11d9e45b49bf98f36eb8a7f357b0dd24f3e7cddef5295518e5bef', - 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', - 'potential-energy-efficiency': '78', 'hot-water-energy-eff': 'Good', - 'low-energy-lighting': '29', - 'walls-description': 'Solid brick, as built, no insulation (assumed)', - 'hotwater-description': 'From main system' - } - - row4 = { - 'UPRN': '100030539619', 'RDSAP_CHANGE': 7, 'HEAT_DEMAND_CHANGE': -41, 'CARBON_CHANGE': -0.5, - 'SAP_STARTING': 66, 'SAP_ENDING': 73, 'HEAT_DEMAND_STARTING': 256, 'HEAT_DEMAND_ENDING': 215, - 'CARBON_STARTING': 4.0, 'CARBON_ENDING': 3.5, 'PROPERTY_TYPE': 'House', 'BUILT_FORM': 'Semi-Detached', - 'CONSTITUENCY': 'E14000909', 'NUMBER_HABITABLE_ROOMS': 5.0, 'NUMBER_HEATED_ROOMS': 5.0, - 'FIXED_LIGHTING_OUTLETS_COUNT': 14.0, 'CONSTRUCTION_AGE_BAND': 'England and Wales: 1930-1949', - 'TRANSACTION_TYPE_STARTING': 'eco assessment', 'MECHANICAL_VENTILATION_STARTING': 'natural', - 'SECONDHEAT_DESCRIPTION_STARTING': 'Room heaters, dual fuel (mineral and wood)', - 'ENERGY_TARIFF_STARTING': 'dual', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N', 'PHOTO_SUPPLY_STARTING': 15.0, - 'GLAZED_TYPE_STARTING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_STARTING': 100.0, - 'LOW_ENERGY_LIGHTING_STARTING': 29.0, 'NUMBER_OPEN_FIREPLACES_STARTING': 1.0, - 'EXTENSION_COUNT_STARTING': 1.0, 'TOTAL_FLOOR_AREA_STARTING': 87.0, 'FLOOR_HEIGHT_STARTING': 2.4, - 'TRANSACTION_TYPE_ENDING': 'eco assessment', 'MECHANICAL_VENTILATION_ENDING': 'natural', - 'SECONDHEAT_DESCRIPTION_ENDING': 'Room heaters, dual fuel (mineral and wood)', - 'ENERGY_TARIFF_ENDING': 'dual', 'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 15.0, - 'GLAZED_TYPE_ENDING': 'double glazing, unknown install date', 'MULTI_GLAZE_PROPORTION_ENDING': 100.0, - 'LOW_ENERGY_LIGHTING_ENDING': 29.0, 'NUMBER_OPEN_FIREPLACES_ENDING': 0, 'EXTENSION_COUNT_ENDING': 1.0, - 'TOTAL_FLOOR_AREA_ENDING': 87.0, 'FLOOR_HEIGHT_ENDING': 2.4, 'DAYS_TO_STARTING': 2887, - 'DAYS_TO_ENDING': 2960, 'walls_thermal_transmittance': 1.7, 'is_cavity_wall': False, - 'is_filled_cavity': False, 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, - 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'is_sandstone_or_limestone': False, - 'is_park_home': False, 'walls_insulation_thickness': 'none', 'external_insulation': False, - 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 1.7, 'is_park_home_ENDING': False, - 'walls_insulation_thickness_ENDING': 'none', 'external_insulation_ENDING': False, - 'internal_insulation_ENDING': False, 'floor_thermal_transmittance': 0.53, 'is_to_unheated_space': False, - 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, 'another_property_below': False, - 'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.53, - 'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.21, 'is_pitched': True, - 'is_roof_room': False, 'is_loft': True, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - 'has_dwelling_above': False, 'roof_insulation_thickness': '200', 'roof_thermal_transmittance_ENDING': 0.21, - 'roof_insulation_thickness_ENDING': '200', 'heater_type': 'Unknown', 'system_type': 'from main system', - 'thermostat_characteristics': 'Unknown', 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', - 'hotwater_tariff_type': 'Unknown', 'extra_features': 'Unknown', 'chp_systems': 'Unknown', - 'distribution_system': 'Unknown', 'no_system_present': 'Unknown', 'appliance': 'Unknown', - 'heater_type_ENDING': 'Unknown', 'system_type_ENDING': 'from main system', - 'thermostat_characteristics_ENDING': 'Unknown', 'heating_scope_ENDING': 'Unknown', - 'energy_recovery_ENDING': 'Unknown', 'hotwater_tariff_type_ENDING': 'Unknown', - 'extra_features_ENDING': 'Unknown', 'chp_systems_ENDING': 'Unknown', - 'distribution_system_ENDING': 'Unknown', 'no_system_present_ENDING': 'Unknown', - 'appliance_ENDING': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, - 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, - 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, - 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, - 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, - 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, - 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, - 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, - 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, - 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, - 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, - 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': True, - 'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False, - 'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False, - 'has_boiler_ENDING': True, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': False, - 'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False, - 'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False, - 'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False, - 'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False, - 'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False, - 'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False, - 'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False, - 'has_electric_ENDING': False, 'has_mains_gas_ENDING': True, 'has_wood_logs_ENDING': False, - 'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False, - 'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False, - 'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False, - 'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False, - 'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'room thermostat', - 'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown', - 'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False, - 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown', - 'thermostatic_control_ENDING': 'room thermostat', 'charging_system_ENDING': 'Unknown', - 'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown', - 'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False, - 'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown', - 'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'mains gas', - 'main-fuel_tariff_type': 'Unknown', 'is_community': False, - 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', - 'fuel_type_ENDING': 'mains gas', 'main-fuel_tariff_type_ENDING': 'Unknown', 'is_community_ENDING': False, - 'no_individual_heating_or_community_network_ENDING': False, 'complex_fuel_type_ENDING': 'Unknown', - 'estimated_perimeter_STARTING': 27.113649698998472, 'estimated_perimeter_ENDING': 27.113649698998472, - 'HOT_WATER_ENERGY_EFF_STARTING': "Good", - "FLOOR_ENERGY_EFF_STARTING": "Unknown", - "WINDOWS_ENERGY_EFF_STARTING": "Average", - "WALLS_ENERGY_EFF_STARTING": "Very Poor", - "SHEATING_ENERGY_EFF_STARTING": "Unknown", - "ROOF_ENERGY_EFF_STARTING": "Good", - "MAINHEAT_ENERGY_EFF_STARTING": "Good", - "MAINHEATC_ENERGY_EFF_STARTING": "Average", - "LIGHTING_ENERGY_EFF_STARTING": "Average", - "POTENTIAL_ENERGY_EFFICIENCY": 78, - "ENVIRONMENT_IMPACT_POTENTIAL": 76, - "ENERGY_CONSUMPTION_POTENTIAL": 153, - "CO2_EMISSIONS_POTENTIAL": 2.4, - 'HOT_WATER_ENERGY_EFF_ENDING': "Good", - "FLOOR_ENERGY_EFF_ENDING": "Unknown", - "WINDOWS_ENERGY_EFF_ENDING": "Average", - "WALLS_ENERGY_EFF_ENDING": "Very Poor", - "SHEATING_ENERGY_EFF_ENDING": "Unknown", - "ROOF_ENERGY_EFF_ENDING": "Good", - "MAINHEAT_ENERGY_EFF_ENDING": "Good", - "MAINHEATC_ENERGY_EFF_ENDING": "Average", - "LIGHTING_ENERGY_EFF_ENDING": "Average", - } - - home4 = Property( - id=0, - postcode=starting_epc4["postcode"], - address=starting_epc4["address1"], - data=starting_epc4 - ) - home4.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) - home4.set_number_lighting_outlets(None) - - data_processor4 = DataProcessor(None, newdata=True) - data_processor4.insert_data(pd.DataFrame([home4.get_model_data()])) - - data_processor4.pre_process() - - starting_epc_data4 = data_processor4.get_component_features(suffix="_STARTING") - ending_epc_data4 = data_processor4.get_component_features(suffix="_ENDING") - fixed_data4 = data_processor4.get_fixed_features() - - ending_lodgement_date4 = '2022-09-08' - - ending_epc_data4["DAYS_TO_ENDING"] = data_processor4.calculate_days_to(ending_lodgement_date4) - - recommendation4 = { - "recommendation_id": 0, - "type": "sealing_open_fireplace" - } - - test_record4 = create_recommendation_scoring_data( - property=home4, - recommendation=recommendation4, - starting_epc_data=starting_epc_data4, - ending_epc_data=ending_epc_data4, - fixed_data=fixed_data4, - ) - test_record4 = pd.DataFrame([test_record4]) - - # Test the final cleaning: - test_record4 = DataProcessor.apply_averages_cleaning( - data_to_clean=test_record4, - cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] - ).drop(columns=["LOCAL_AUTHORITY"]) - - test_record4 = DataProcessor.clean_missings_after_description_process( - test_record4, [ - c for c in test_record4.columns if - ("thermal_transmittance" in c) or ("insulation_thickness" in c) - ] - ) - - for c in test_record4.columns: - if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]: - continue - - assert test_record4[c].values[0] == row4[c] diff --git a/etl/epc/tests/test_epcrecord.py b/etl/epc/tests/test_epcrecord.py index 48ad5148..cf0361b1 100644 --- a/etl/epc/tests/test_epcrecord.py +++ b/etl/epc/tests/test_epcrecord.py @@ -109,14 +109,13 @@ class TestEpcRecord: assert record.prepared_epc["energy-consumption-current"] == 200.0 assert record.prepared_epc["co2-emissions-current"] == 5.5 - def test_clean_energy_empty_values(self, cleaning_data, epc_records_1): + def test_clean_energy_empty_values(self, cleaning_data): # We cannot have invalid values so this should raise an exception record = EPCRecord(cleaning_data=cleaning_data) record.prepared_epc = { "energy-consumption-current": "", "co2-emissions-current": "" } - record._clean_energy() with pytest.raises(ValueError): record._clean_energy() @@ -207,7 +206,7 @@ class TestEpcRecord: "mains-gas-flag": "InvalidValue" } # It should always be Y or N or an anomally value - with pytest.raises(ValueError): + with pytest.raises(KeyError): record._clean_mains_gas() record = EPCRecord(cleaning_data=cleaning_data) @@ -225,7 +224,8 @@ class TestEpcRecord: } record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is True + assert record.prepared_epc["solar-water-heating-flag"] == "Y" + assert record.solar_water_heating_flag_bool is True def test_clean_solar_hot_water_empty(self, cleaning_data): record = EPCRecord(cleaning_data=cleaning_data) @@ -234,7 +234,8 @@ class TestEpcRecord: } record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is None + assert record.prepared_epc["solar-water-heating-flag"] == "N" + assert record.solar_water_heating_flag_bool is False def test_clean_number_lighting_outlets_valid(self, cleaning_data, epc_records_1): record = EPCRecord(cleaning_data=cleaning_data, epc_records=epc_records_1) @@ -320,7 +321,8 @@ class TestEpcRecord: record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is True + assert record.prepared_epc["solar-water-heating-flag"] == "Y" + assert record.solar_water_heating_flag_bool is True record = EPCRecord(cleaning_data=cleaning_data) @@ -330,7 +332,8 @@ class TestEpcRecord: record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is False + assert record.prepared_epc["solar-water-heating-flag"] == "N" + assert record.solar_water_heating_flag_bool is False record = EPCRecord(cleaning_data=cleaning_data) @@ -340,7 +343,8 @@ class TestEpcRecord: record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is None + assert record.prepared_epc["solar-water-heating-flag"] == "N" + assert record.solar_water_heating_flag_bool is False record = EPCRecord(cleaning_data=cleaning_data) @@ -350,4 +354,5 @@ class TestEpcRecord: record._clean_solar_hot_water() - assert record.prepared_epc["solar-water-heating-flag"] is None + assert record.prepared_epc["solar-water-heating-flag"] == "N" + assert record.solar_water_heating_flag_bool is False