Added tests for solid walls

This commit is contained in:
Khalim Conn-Kowlessar 2023-10-17 14:13:42 +11:00
parent 482ddb1e98
commit 13490b18ca
2 changed files with 292 additions and 87 deletions

View file

@ -9,93 +9,100 @@ import pytest
import pickle
from tqdm import tqdm
with open("sap_change_dataset.pickle", "rb") as f:
sap_change_dataset = pickle.load(f)
# import pickle
#
# with open("sap_change_dataset.pickle", "rb") as f:
# sap_change_dataset = pickle.load(f)
#
# search_from = sap_change_dataset[
# sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]
# ]
# search_from = search_from[
# (search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) &
# (search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) &
# (search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) &
# (search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) &
# (search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"])
# ]
#
# # Find a record where the only difference is cavity wall getting filled
# ending_cols = [c for c in search_from.columns if "_ENDING" in c]
#
# ignore = [
# "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING",
# "DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING"
# ]
#
# ending_cols = [c for c in ending_cols if c not in ignore]
#
# for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]):
#
# same = True
# starting_cols = []
# for c in ending_cols:
#
# starting_col = c.replace("_ENDING", "")
# if starting_col not in search_from.columns:
# starting_col = c.replace("_ENDING", "_STARTING")
# if starting_col not in search_from.columns:
# raise Exception("something went wrong")
#
# starting_cols.append(starting_col)
#
# # We want them to be different
# if c == "walls_thermal_transmittance_ENDING":
# if row[c] == row[starting_col]:
# same = False
# break
# else:
# continue
#
# # We now check if the starting and ending values are the same
# if row[c] != row[starting_col]:
# same = False
# break
#
# if same:
# raise Exception("We found one!")
#
# fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols]
#
# import pandas as pd
#
# start = row[starting_cols]
# start.index = [c.replace("_STARTING", "") for c in start.index]
# end = row[ending_cols]
# end.index = [c.replace("_ENDING", "") for c in end.index]
# start["type"] = "starting"
# end["type"] = "ending"
#
# compare = pd.concat([start, end], axis=1)
#
# ending_lmk = "1481856809222016121510000597528546"
# starting_lmk = "1481856849902016092320290148762028"
#
# EPC_AUTH_TOKEN = "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA="
#
# client = EpcClient(auth_token=EPC_AUTH_TOKEN)
# result = client.domestic.search(params={"address": "26, Vicarage Lane", "postcode": "NG32 1SP"})
# starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0]
# ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0]
search_from = sap_change_dataset[
(sap_change_dataset["walls_thermal_transmittance_ENDING"] != sap_change_dataset["walls_thermal_transmittance"]) &
(sap_change_dataset["is_solid_brick"])
]
search_from = search_from[
(search_from["roof_thermal_transmittance_ENDING"] == search_from["roof_thermal_transmittance"]) &
(search_from["floor_thermal_transmittance_ENDING"] == search_from["floor_thermal_transmittance"]) &
(search_from["MECHANICAL_VENTILATION_ENDING"] == search_from["MECHANICAL_VENTILATION_STARTING"]) &
(search_from["SECONDHEAT_DESCRIPTION_ENDING"] == search_from["SECONDHEAT_DESCRIPTION_STARTING"]) &
(search_from["GLAZED_TYPE_ENDING"] == search_from["GLAZED_TYPE_STARTING"])
]
# Find a record where the only difference is cavity wall getting filled
ending_cols = [c for c in search_from.columns if "_ENDING" in c]
ignore = [
"SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING", "TRANSACTION_TYPE_ENDING", "FLOOR_HEIGHT_ENDING",
"DAYS_TO_ENDING", "TOTAL_FLOOR_AREA_ENDING"
]
ending_cols = [c for c in ending_cols if c not in ignore]
for _, row in tqdm(search_from.iterrows(), total=search_from.shape[0]):
same = True
starting_cols = []
for c in ending_cols:
starting_col = c.replace("_ENDING", "")
if starting_col not in search_from.columns:
starting_col = c.replace("_ENDING", "_STARTING")
if starting_col not in search_from.columns:
raise Exception("something went wrong")
starting_cols.append(starting_col)
# We want them to be different
if c == "walls_thermal_transmittance_ENDING":
if row[c] == row[starting_col]:
same = False
break
else:
continue
# We want them to be different
if c == "walls_insulation_thickness_ENDING":
if row[c] == row[starting_col]:
same = False
break
else:
continue
# We now check if the starting and ending values are the same
if row[c] != row[starting_col]:
same = False
break
if same:
raise Exception("We found one!")
fixed_cols = [c for c in search_from.columns if c not in starting_cols + ending_cols]
import pandas as pd
start = row[starting_cols]
start.index = [c.replace("_STARTING", "") for c in start.index]
end = row[ending_cols]
end.index = [c.replace("_ENDING", "") for c in end.index]
start["type"] = "starting"
end["type"] = "ending"
compare = pd.concat([start, end], axis=1)
ending_lmk = "d0fc64d6b80db04c32998c9b846dd04c8f0b486231a11e4c062020b35af1312d"
starting_lmk = "b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d"
client = EpcClient(auth_token=EPC_AUTH_TOKEN)
result = client.domestic.search(params={"address": "FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY", "postcode": "E16 1BD"})
starting_epc = [x for x in result["rows"] if x["lmk-key"] == starting_lmk][0]
ending_epc = [x for x in result["rows"] if x["lmk-key"] == ending_lmk][0]
# with open("cleaned.pickle", "wb") as f:
# pickle.dump(cleaned, f)
# with open(
# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaned.pickle", "rb"
# ) as f:
# cleaned = pickle.load(f)
# with open("cleaning_data.pickle", "wb") as f:
# pickle.dump(cleaning_data, f)
# with open(
# os.path.abspath(os.path.dirname(__file__)) + "/backend/tests/test_data/cleaning_data.pickle", "rb"
# ) as f:
# cleaning_data = pickle.load(f)
class TestSapModelPrep:
@ -327,5 +334,201 @@ class TestSapModelPrep:
assert test_record[c].values[0] == row[c]
def test_solid_wall_insulation(self):
pass
def test_solid_wall_insulation(self, cleaned, cleaning_data):
starting_epc2 = {
'low-energy-fixed-light-count': '2', 'address': 'FLAT 12, WAREHOUSE W, 3 WESTERN GATEWAY',
'uprn-source': 'Energy Assessor', 'floor-height': '3.64', 'heating-cost-potential': '465',
'unheated-corridor-length': '', 'hot-water-cost-potential': '185',
'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C',
'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Poor',
'environment-impact-potential': '51', 'glazed-type': 'double glazing installed during or after 2002',
'heating-cost-current': '1223', 'address3': '3 WESTERN GATEWAY',
'mainheatcont-description': 'Programmer and appliance thermostats', 'sheating-energy-eff': 'N/A',
'property-type': 'Flat', 'local-authority-label': 'Newham', 'fixed-lighting-outlets-count': '12',
'energy-tariff': 'off-peak 7 hour', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '342',
'county': '', 'postcode': 'E16 1BD', 'solar-water-heating-flag': 'N', 'constituency': 'E14001032',
'co2-emissions-potential': '3.6', 'number-heated-rooms': '2', 'floor-description': '(other premises below)',
'energy-consumption-potential': '307', 'local-authority': 'E09000025', 'built-form': 'Mid-Terrace',
'number-open-fireplaces': '0', 'windows-description': 'Partial double glazing', 'glazed-area': 'Normal',
'inspection-date': '2020-10-14', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '66',
'address1': 'FLAT 12', 'heat-loss-corridor': 'heated corridor', 'flat-storey-count': '',
'constituency-label': 'West Ham', 'roof-energy-eff': 'N/A', 'total-floor-area': '70.0',
'building-reference-number': '10000539740', 'environment-impact-current': '42',
'co2-emissions-current': '4.6', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A',
'number-habitable-rooms': '2', 'address2': 'WAREHOUSE W', 'hot-water-env-eff': 'Poor', 'posttown': 'LONDON',
'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Poor',
'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
'lighting-description': 'Low energy lighting in 17% of fixed outlets', 'roof-env-eff': 'N/A',
'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '67',
'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '61', 'main-heating-controls': '',
'lodgement-datetime': '2020-10-14 00:00:00', 'flat-top-storey': 'N', 'current-energy-rating': 'F',
'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'marketed sale',
'uprn': '10012839482', 'current-energy-efficiency': '33', 'energy-consumption-current': '393',
'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '110',
'lodgement-date': '2020-10-14', 'extension-count': '0', 'mainheatc-env-eff': 'Good',
'lmk-key': 'b0d82f468273bec55ec5676a809b8e36b55db940ffa92f482a482f6aaa38eb1d', 'wind-turbine-count': '0',
'tenure': 'Owner-occupied', 'floor-level': '01', 'potential-energy-efficiency': '71',
'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '17',
'walls-description': 'Solid brick, as built, no insulation (assumed)',
'hotwater-description': 'Electric immersion, standard tariff'
}
row2 = {
'UPRN': '10012839482', 'RDSAP_CHANGE': 8, 'HEAT_DEMAND_CHANGE': -59,
'CARBON_CHANGE': -0.5999999999999996, 'SAP_STARTING': 33, 'SAP_ENDING': 41, 'HEAT_DEMAND_STARTING': 393,
'HEAT_DEMAND_ENDING': 334, 'CARBON_STARTING': 4.6, 'CARBON_ENDING': 4.0, 'PROPERTY_TYPE': 'Flat',
'BUILT_FORM': 'Mid-Terrace', 'CONSTITUENCY': 'E14001032', 'NUMBER_HABITABLE_ROOMS': 2.0,
'NUMBER_HEATED_ROOMS': 2.0, 'FIXED_LIGHTING_OUTLETS_COUNT': 12.0,
'CONSTRUCTION_AGE_BAND': 'England and Wales: 1996-2002', 'TRANSACTION_TYPE_STARTING': 'marketed sale',
'MECHANICAL_VENTILATION_STARTING': 'natural', 'SECONDHEAT_DESCRIPTION_STARTING': 'None',
'ENERGY_TARIFF_STARTING': 'off-peak 7 hour', 'SOLAR_WATER_HEATING_FLAG_STARTING': 'N',
'PHOTO_SUPPLY_STARTING': 0.0, 'GLAZED_TYPE_STARTING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_STARTING': 61.0, 'LOW_ENERGY_LIGHTING_STARTING': 17.0,
'NUMBER_OPEN_FIREPLACES_STARTING': 0.0, 'EXTENSION_COUNT_STARTING': 0.0,
'TOTAL_FLOOR_AREA_STARTING': 70.0, 'FLOOR_HEIGHT_STARTING': 3.64,
'TRANSACTION_TYPE_ENDING': 'marketed sale', 'MECHANICAL_VENTILATION_ENDING': 'natural',
'SECONDHEAT_DESCRIPTION_ENDING': 'None', 'ENERGY_TARIFF_ENDING': 'off-peak 7 hour',
'SOLAR_WATER_HEATING_FLAG_ENDING': 'N', 'PHOTO_SUPPLY_ENDING': 0.0,
'GLAZED_TYPE_ENDING': 'double glazing installed during or after 2002',
'MULTI_GLAZE_PROPORTION_ENDING': 61.0, 'LOW_ENERGY_LIGHTING_ENDING': 17.0,
'NUMBER_OPEN_FIREPLACES_ENDING': 0.0, 'EXTENSION_COUNT_ENDING': 0.0, 'TOTAL_FLOOR_AREA_ENDING': 70.0,
'FLOOR_HEIGHT_ENDING': 3.64, 'DAYS_TO_STARTING': 2266, 'DAYS_TO_ENDING': 2307,
'walls_thermal_transmittance': 0.45, 'is_cavity_wall': False, 'is_filled_cavity': False,
'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False,
'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False,
'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none',
'external_insulation': False, 'internal_insulation': False, 'walls_thermal_transmittance_ENDING': 0.21,
'is_park_home_ENDING': False, 'walls_insulation_thickness_ENDING': 'average',
'external_insulation_ENDING': False, 'internal_insulation_ENDING': False,
'floor_thermal_transmittance': 0.0, 'is_to_unheated_space': False, 'is_to_external_air': False,
'is_suspended': False, 'is_solid': False, 'another_property_below': True,
'floor_insulation_thickness': 'none', 'floor_thermal_transmittance_ENDING': 0.0,
'floor_insulation_thickness_ENDING': 'none', 'roof_thermal_transmittance': 0.0, 'is_pitched': False,
'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
'has_dwelling_above': True, 'roof_insulation_thickness': 'none',
'roof_thermal_transmittance_ENDING': 0.0, 'roof_insulation_thickness_ENDING': 'none',
'heater_type': 'electric immersion', 'system_type': 'Unknown', 'thermostat_characteristics': 'Unknown',
'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'standard tariff',
'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown',
'no_system_present': 'Unknown', 'appliance': 'Unknown', 'heater_type_ENDING': 'electric immersion',
'system_type_ENDING': 'Unknown', 'thermostat_characteristics_ENDING': 'Unknown',
'heating_scope_ENDING': 'Unknown', 'energy_recovery_ENDING': 'Unknown',
'hotwater_tariff_type_ENDING': 'standard tariff', 'extra_features_ENDING': 'Unknown',
'chp_systems_ENDING': 'Unknown', 'distribution_system_ENDING': 'Unknown',
'no_system_present_ENDING': 'Unknown', 'appliance_ENDING': 'Unknown', 'has_radiators': False,
'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False,
'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False,
'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False,
'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': True,
'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False,
'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_radiators_ENDING': False,
'has_fan_coil_units_ENDING': False, 'has_pipes_in_screed_above_insulation_ENDING': False,
'has_pipes_in_insulated_timber_floor_ENDING': False, 'has_pipes_in_concrete_slab_ENDING': False,
'has_boiler_ENDING': False, 'has_air_source_heat_pump_ENDING': False, 'has_room_heaters_ENDING': True,
'has_electric_storage_heaters_ENDING': False, 'has_warm_air_ENDING': False,
'has_electric_underfloor_heating_ENDING': False, 'has_electric_ceiling_heating_ENDING': False,
'has_community_scheme_ENDING': False, 'has_ground_source_heat_pump_ENDING': False,
'has_no_system_present_ENDING': False, 'has_portable_electric_heaters_ENDING': False,
'has_water_source_heat_pump_ENDING': False, 'has_electric_heat_pump_ENDING': False,
'has_micro-cogeneration_ENDING': False, 'has_solar_assisted_heat_pump_ENDING': False,
'has_exhaust_source_heat_pump_ENDING': False, 'has_community_heat_pump_ENDING': False,
'has_electric_ENDING': True, 'has_mains_gas_ENDING': False, 'has_wood_logs_ENDING': False,
'has_coal_ENDING': False, 'has_oil_ENDING': False, 'has_wood_pellets_ENDING': False,
'has_anthracite_ENDING': False, 'has_dual_fuel_mineral_and_wood_ENDING': False,
'has_smokeless_fuel_ENDING': False, 'has_lpg_ENDING': False, 'has_b30k_ENDING': False,
'has_electricaire_ENDING': False, 'has_assumed_for_most_rooms_ENDING': False,
'has_underfloor_heating_ENDING': False, 'thermostatic_control': 'appliance thermostats',
'charging_system': 'Unknown', 'switch_system': 'programmer', 'no_control': 'Unknown',
'dhw_control': 'Unknown', 'community_heating': 'Unknown', 'multiple_room_thermostats': False,
'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', 'rate_control': 'Unknown',
'thermostatic_control_ENDING': 'appliance thermostats', 'charging_system_ENDING': 'Unknown',
'switch_system_ENDING': 'programmer', 'no_control_ENDING': 'Unknown', 'dhw_control_ENDING': 'Unknown',
'community_heating_ENDING': 'Unknown', 'multiple_room_thermostats_ENDING': False,
'auxiliary_systems_ENDING': 'Unknown', 'trvs_ENDING': 'Unknown', 'rate_control_ENDING': 'Unknown',
'glazing_type': 'double', 'glazing_type_ENDING': 'double', 'fuel_type': 'electricity',
'main-fuel_tariff_type': 'Unknown', 'is_community': False,
'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown',
'fuel_type_ENDING': 'electricity', 'main-fuel_tariff_type_ENDING': 'Unknown',
'is_community_ENDING': False, 'no_individual_heating_or_community_network_ENDING': False,
'complex_fuel_type_ENDING': 'Unknown', 'estimated_perimeter_STARTING': 35.4964786985977,
'estimated_perimeter_ENDING': 35.4964786985977
}
home2 = Property(
id=0,
postcode=starting_epc2["postcode"],
address1=starting_epc2["address1"],
epc_client=EpcClient(auth_token="notoken"),
data=starting_epc2
)
home2.get_components(cleaned)
data_processor2 = DataProcessor(None, newdata=True)
data_processor2.insert_data(pd.DataFrame([home2.get_model_data()]))
data_processor2.pre_process()
starting_epc_data2 = data_processor2.get_component_features(suffix="_STARTING")
ending_epc_data2 = data_processor2.get_component_features(suffix="_ENDING")
fixed_data2 = data_processor2.get_fixed_features()
ending_lodgement_date2 = '2020-11-24'
starting_epc_data2["DAYS_TO_ENDING"] = data_processor2.calculate_days_to(ending_lodgement_date2)
recommendation2 = {
"recommendation_id": 0,
"new_u_value": 0.21,
"type": "wall_insulation"
}
test_record2 = create_recommendation_scoring_data(
property=home2,
recommendation=recommendation2,
starting_epc_data=starting_epc_data2,
ending_epc_data=ending_epc_data2,
fixed_data=fixed_data2,
)
test_record2 = pd.DataFrame([test_record2])
# Test the final cleaning:
test_record2 = DataProcessor.apply_averages_cleaning(
data_to_clean=test_record2,
cleaning_data=cleaning_data,
cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"]
).drop(columns=["LOCAL_AUTHORITY"])
test_record2 = DataProcessor.clean_missings_after_description_process(
test_record2, [
c for c in test_record2.columns if
("thermal_transmittance" in c) or ("insulation_thickness" in c)
]
)
for c in test_record2.columns:
if c in ["id", "SAP_ENDING", "HEAT_DEMAND_ENDING", "CARBON_ENDING"]:
continue
if c == "FLOOR_HEIGHT_ENDING":
assert (row2[c] - test_record2[c].values[0]) <= 0.020001
continue
if c == "walls_insulation_thickness_ENDING":
assert row2[c] == "average"
assert test_record2[c].values[0] == "above average"
continue
if c == "CONSTRUCTION_AGE_BAND":
# For this, we have different values in the original data
assert row2[c] == "England and Wales: 1996-2002"
assert test_record2[c].values[0] == "England and Wales: 1900-1929"
continue
assert test_record2[c].values[0] == row2[c]

View file

@ -499,10 +499,12 @@ def app():
# Add some temporal features - we look at the days from the standard starting point in time
# for the starting and ending date so all records are from a fixed point
data_by_urpn_df["DAYS_TO_STARTING"] = DataProcessor.calculate_days_to(
data_by_urpn_df["LODGEMENT_DATE_STARTING"])
data_by_urpn_df["LODGEMENT_DATE_STARTING"]
)
data_by_urpn_df["DAYS_TO_ENDING"] = DataProcessor.calculate_days_to(
data_by_urpn_df["LODGEMENT_DATE_ENDING"])
data_by_urpn_df["LODGEMENT_DATE_ENDING"]
)
data_by_urpn_df = data_by_urpn_df.drop(columns=["LODGEMENT_DATE_STARTING", "LODGEMENT_DATE_ENDING"])