import pandas as pd import msgpack from datetime import datetime from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 from backend.Property import Property from recommendations.HeatingRecommender import HeatingRecommender from recommendations.Recommendations import Recommendations from etl.epc.Record import EPCRecord from etl.solar.SolarPhotoSupply import SolarPhotoSupply from backend.ml_models.api import ModelApi def find_examples(): """ Some scrappy helper code to find EPC examples""" # Let's look for some testing data, where the only thing different pre and post is the installation of an # air source heat pump data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/2024-03-24-15-51-13/dataset_no_cleaning.parquet" ) # Firstly, take records where before there was no air source heat pump and afterwards there was data = data[ data["has_air_source_heat_pump_ending"] & ~data["has_air_source_heat_pump"] ] # Start with a property that has a boiler data = data[data["has_boiler"]] static_columns = [ # Walls 'walls_thermal_transmittance_ending', 'is_filled_cavity_ending', 'is_park_home_ending', 'walls_insulation_thickness_ending', 'external_insulation_ending', 'internal_insulation_ending', # Floors # 'floor_thermal_transmittance_ending', # Don't subset on this, because it changes based on floor area 'floor_insulation_thickness_ending', # Roof 'roof_thermal_transmittance_ending', 'is_at_rafters_ending', 'roof_insulation_thickness_ending', # Hot water - air source heat pump will shange the hot water system (probably from whatever it was -> main) # 'heater_type_ending', # 'system_type_ending', # 'thermostat_characteristics_ending', # 'heating_scope_ending', # 'energy_recovery_ending', # 'hotwater_tariff_type_ending', # 'extra_features_ending', # 'chp_systems_ending', # 'distribution_system_ending', # 'no_system_present_ending', # 'appliance_ending', # Heating - Will change when installing an ASHP # 'has_radiators_ending', # 'has_fan_coil_units_ending', # 'has_pipes_in_screed_above_insulation_ending', # 'has_pipes_in_insulated_timber_floor_ending', # 'has_pipes_in_concrete_slab_ending', # 'has_boiler_ending', # 'has_air_source_heat_pump_ending', # We want the air source heat pump to change # 'has_room_heaters_ending', # 'has_electric_storage_heaters_ending', # 'has_warm_air_ending', # 'has_electric_underfloor_heating_ending', # 'has_electric_ceiling_heating_ending', # 'has_community_scheme_ending', # 'has_ground_source_heat_pump_ending', # 'has_no_system_present_ending', # 'has_portable_electric_heaters_ending', # 'has_water_source_heat_pump_ending', # 'has_electric_heat_pump_ending', # 'has_micro-cogeneration_ending', # 'has_solar_assisted_heat_pump_ending', # 'has_exhaust_source_heat_pump_ending', # 'has_community_heat_pump_ending', # 'has_electric_ending', # 'has_mains_gas_ending', # 'has_wood_logs_ending', 'has_coal_ending', 'has_oil_ending', # 'has_wood_pellets_ending', 'has_anthracite_ending', 'has_dual_fuel_mineral_and_wood_ending', # 'has_smokeless_fuel_ending', 'has_lpg_ending', 'has_b30k_ending', 'has_electricaire_ending', # 'has_assumed_for_most_rooms_ending', 'has_underfloor_heating_ending', # 'thermostatic_control_ending', # 'charging_system_ending', # 'switch_system_ending', # 'no_control_ending', # 'dhw_control_ending', # 'community_heating_ending', # 'multiple_room_thermostats_ending', # 'auxiliary_systems_ending', # 'trvs_ending', # 'rate_control_ending', # Window 'glazing_type_ending', # Fuel - could change with ASHP # 'fuel_type_ending', # 'main-fuel_tariff_type_ending', # 'is_community_ending', # 'no_individual_heating_or_community_network_ending', # 'complex_fuel_type_ending', 'mechanical_ventilation_ending', 'secondheat_description_ending', 'glazed_type_ending', 'multi_glaze_proportion_ending', 'low_energy_lighting_ending', 'number_open_fireplaces_ending', 'solar_water_heating_flag_ending', 'photo_supply_ending', 'energy_tariff_ending', 'extension_count_ending', 'total_floor_area_ending', # 'hot_water_energy_eff_ending', 'floor_energy_eff_ending', 'windows_energy_eff_ending', 'walls_energy_eff_ending', 'sheating_energy_eff_ending', 'roof_energy_eff_ending', # 'mainheat_energy_eff_ending', # 'mainheatc_energy_eff_ending', 'lighting_energy_eff_ending', 'number_habitable_rooms_ending', 'number_heated_rooms_ending', ] for col in static_columns: base_starting = col.split("_ending")[0] if base_starting + "_starting" in data.columns: starting_col = base_starting + "_starting" else: starting_col = base_starting # Filter print("Column: %s" % col) print("Starting size: %s" % data.shape[0]) data = data[data[starting_col] == data[col]] print("Ending size: %s" % data.shape[0]) z = data[['uprn', col, starting_col]] # Great example UPRNs # 100030969273 # 10034685399 - Completely transforms the heating and hot water systems in the home (goes from oil -> electricity) # 100091200828 - goes from a liquid petroleum gas boiler to ashp # Look for starting with a gas boiler data[ data["has_boiler"] & data["has_radiators"] & data["has_mains_gas"] & ~data["has_boiler_ending"] ] # UPRN: 100011776843 class TestAirSourceHeatPump: def test_eligible(self): # This tests a house, which will be suitable for an air source heat pump epc_record = EPCRecord() epc_record.prepared_epc = { "county": "Broxbourne", "mainheat-energy-eff": "Good", "hot-water-energy-eff": "Good", "mainheatc-energy-eff": "Good", "number-heated-rooms": 5, "property-type": "House", "built-form": "Semi-Detached" } property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) property_instance.main_heating = { 'original_description': 'Boiler and radiators, mains gas', "clean_description": "Boiler and radiators, mains gas", 'has_radiators': True, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, "has_electric_heat_pumps": False, "has_micro-cogeneration": False } property_instance.main_fuel = { 'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas', 'tariff_type': None, 'is_community': False, 'no_individual_heating_or_community_network': False, 'complex_fuel_type': None } property_instance.hotwater = { 'original_description': 'From main system', 'clean_description': 'From main system', 'heater_type': None, 'system_type': 'from main system', 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None } property_instance.main_heating_controls = { 'original_description': 'Programmer, room thermostat and TRVs', 'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer', 'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, 'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None } recommender = HeatingRecommender(property_instance=property_instance) assert not recommender.heating_recommendations recommender.recommend(phase=0) assert recommender.recommendation is None def test_air_source_heat_pump_gas_boiler_starting(self): starting_epc = { 'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor', 'floor-height': '2.62', 'heating-cost-potential': '599', 'unheated-corridor-length': '', 'hot-water-cost-potential': '67', 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '72', 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '913', 'address3': '', 'mainheatcont-description': 'Programmer, no room thermostat', 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Wigan', 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '210', 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', 'constituency': 'E14001039', 'co2-emissions-potential': '2.6', 'number-heated-rooms': '4', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '180', 'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-02-15', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '78', 'address1': '430 Gidlow Lane', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112', 'environment-impact-current': '38', 'co2-emissions-current': '6.2', 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN', 'mainheatc-energy-eff': 'Very Poor', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '67', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2022-02-23 16:39:41', 'flat-top-storey': '', 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843', 'current-energy-efficiency': '45', 'energy-consumption-current': '441', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '67', 'lodgement-date': '2022-02-23', 'extension-count': '1', 'mainheatc-env-eff': 'Very Poor', 'lmk-key': '46cb404438a6d88ddff8965cab8b3027ec15c32d93e0b6a5f0381a5109b9bb0d', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '77', 'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '100', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system, no cylinder thermostat' } ending_epc = { 'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor', 'floor-height': '2.62', 'heating-cost-potential': '803', 'unheated-corridor-length': '', 'hot-water-cost-potential': '292', 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '78', 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '861', 'address3': '', 'mainheatcont-description': 'Time and temperature zone control', 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Wigan', 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '434', 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', 'constituency': 'E14001039', 'co2-emissions-potential': '2.0', 'number-heated-rooms': '4', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '147', 'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-05-11', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '43', 'address1': '430 Gidlow Lane', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112', 'environment-impact-current': '63', 'co2-emissions-current': '3.4', 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN', 'mainheatc-energy-eff': 'Very Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '67', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2022-06-06 13:01:20', 'flat-top-storey': '', 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843', 'current-energy-efficiency': '53', 'energy-consumption-current': '252', 'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '67', 'lodgement-date': '2022-06-06', 'extension-count': '1', 'mainheatc-env-eff': 'Very Good', 'lmk-key': '672d5947f3d4a55d97255af71651d6127a939418fa66a687070af77e0ba90df2', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '70', 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '100', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } # differences = [] # for k, v in ending_epc.items(): # if v != starting_epc[k]: # differences.append( # { # "variable": k, # "starting_value": starting_epc[k], # "ending_value": v # } # ) # differences = pd.DataFrame(differences) # # diffs = differences[ # differences["variable"].isin( # [ # "mainheat-energy-eff", # "mainheatcont-description", # "mainheatc-energy-eff", # "main-fuel", # "mainheat-env-eff", # "mainheat-description", # "hot-water-energy-eff", # "hotwater-description" # ] # ) # ] cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = HeatingRecommender(property_instance=home) recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) # Patch - for this property, the hot water energy efficiency is very poor. it's not clear why this is, # but we insert this for this test recommender.heating_recommendations[0]["simulation_config"]["hot_water_energy_eff_ending"] = "Very Poor" property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) assert len(recommender.heating_recommendations) == 1 home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 52.2 def test_air_source_heat_pump_gas_boiler_starting_2(self): """ This property seems to have miniscule movement in SAP - just 2 poins :return: """ starting_epc = { 'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', 'uprn-source': 'Energy Assessor', 'floor-height': '2.3', 'heating-cost-potential': '394', 'unheated-corridor-length': '', 'hot-water-cost-potential': '48', 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good', 'environment-impact-potential': '87', 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '487', 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', 'fixed-lighting-outlets-count': '5', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '86', 'county': '', 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614', 'co2-emissions-potential': '0.8', 'number-heated-rooms': '2', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '105', 'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-25', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '56', 'address1': '31 Whinney Hill Park', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Calder Valley', 'roof-energy-eff': 'Good', 'total-floor-area': '44.0', 'building-reference-number': '10001772583', 'environment-impact-current': '62', 'co2-emissions-current': '2.5', 'roof-description': 'Pitched, 250 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '2', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'BRIGHOUSE', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2021-11-25 11:39:35', 'flat-top-storey': '', 'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '62', 'energy-consumption-current': '322', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '56', 'lodgement-date': '2021-11-25', 'extension-count': '0', 'mainheatc-env-eff': 'Good', 'lmk-key': '077f70657e9c3f1f0ce5392798398398616b159493b2a8ca2338961596631c27', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '60', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } ending_epc = { 'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', 'uprn-source': 'Energy Assessor', 'floor-height': '2.3', 'heating-cost-potential': '277', 'unheated-corridor-length': '', 'hot-water-cost-potential': '266', 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good', 'environment-impact-potential': '90', 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '331', 'address3': '', 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', 'fixed-lighting-outlets-count': '5', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '404', 'county': '', 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614', 'co2-emissions-potential': '0.7', 'number-heated-rooms': '2', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '92', 'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-25', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '48', 'address1': '31 Whinney Hill Park', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Calder Valley', 'roof-energy-eff': 'Good', 'total-floor-area': '44.0', 'building-reference-number': '10001772583', 'environment-impact-current': '68', 'co2-emissions-current': '2.1', 'roof-description': 'Pitched, 250 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '2', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'BRIGHOUSE', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2022-03-23 16:06:21', 'flat-top-storey': '', 'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '64', 'energy-consumption-current': '283', 'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '57', 'lodgement-date': '2022-03-23', 'extension-count': '0', 'mainheatc-env-eff': 'Average', 'lmk-key': '6296248141447b53426a40f1c39da17dad5f4786485db55ee38737891111a4d4', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '89', 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '60', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } # differences = [] # for k, v in ending_epc.items(): # if v != starting_epc[k]: # differences.append( # { # "variable": k, # "starting_value": starting_epc[k], # "ending_value": v # } # ) # differences = pd.DataFrame(differences) # # diffs = differences[ # differences["variable"].isin( # [ # "mainheat-energy-eff", # "mainheatcont-description", # "mainheatc-energy-eff", # "main-fuel", # "mainheat-env-eff", # "mainheat-description", # "hot-water-energy-eff", # "hotwater-description" # ] # ) # ] cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = HeatingRecommender(property_instance=home) recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) assert len(recommender.heating_recommendations) == 1 home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 69.3 # In actuality with this property, the heating controls get downgraded, so we test a manual patch of this patched_simulation_config = { 'mainheat_energy_eff_ending': "Very Good", 'hot_water_energy_eff_ending': 'Very Poor', 'has_boiler_ending': False, 'has_air_source_heat_pump_ending': True, 'has_electric_ending': True, 'has_mains_gas_ending': False, 'fuel_type_ending': 'electricity', 'trvs_ending': None, "mainheatc_energy_eff_ending": 'Average' } # PATCHING property_recommendations_patch = Recommendations.insert_temp_recommendation_id( [recommender.heating_recommendations] ) property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations_patch, [] ) scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict_patch = model_api.predict_all( df=scoring_data_patch, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) # The error is only 0.3, so the model is working assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 64.3 assert ending_epc["current-energy-efficiency"] == '64' def test_air_source_heat_pump_lpg_boiler(self): starting_epc = { 'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry', 'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '1628', 'unheated-corridor-length': '', 'hot-water-cost-potential': '175', 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'D', 'mainheat-energy-eff': 'Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', 'environment-impact-potential': '70', 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '2158', 'address3': 'Perry', 'mainheatcont-description': 'No time or thermostatic control of room temperature', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire', 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '257', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '3.3', 'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '128', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2023-08-31', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '51', 'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0', 'building-reference-number': '10005199915', 'environment-impact-current': '50', 'co2-emissions-current': '5.9', 'roof-description': 'Pitched, 270 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive', 'hot-water-env-eff': 'Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Very Poor', 'main-fuel': 'LPG (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '166', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2023-10-30 13:46:54', 'flat-top-storey': '', 'current-energy-rating': 'F', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '32', 'energy-consumption-current': '243', 'mainheat-description': 'Boiler and radiators, LPG', 'lighting-cost-current': '277', 'lodgement-date': '2023-10-30', 'extension-count': '0', 'mainheatc-env-eff': 'Very Poor', 'lmk-key': 'f1d3bd4b8b50bc9b006231ccb158537c408523b748b3f4ef7e98cd03b144afa5', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '56', 'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '33', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } ending_epc = { 'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry', 'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '917', 'unheated-corridor-length': '', 'hot-water-cost-potential': '328', 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'A', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', 'environment-impact-potential': '96', 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '1098', 'address3': 'Perry', 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire', 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '328', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '0.3', 'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '16', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2023-10-05', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '6', 'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0', 'building-reference-number': '10005199915', 'environment-impact-current': '92', 'co2-emissions-current': '0.7', 'roof-description': 'Pitched, 270 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive', 'hot-water-env-eff': 'Very Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good', 'walls-energy-eff': 'Average', 'photo-supply': '', 'lighting-cost-potential': '166', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2023-11-01 16:29:16', 'flat-top-storey': '', 'current-energy-rating': 'A', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '92', 'energy-consumption-current': '37', 'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '277', 'lodgement-date': '2023-11-01', 'extension-count': '0', 'mainheatc-env-eff': 'Average', 'lmk-key': 'cb7f2838b727907767c8c2a385cd22f722b1e4745463391d910d228e52124515', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '95', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '33', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = HeatingRecommender(property_instance=home) recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) assert len(recommender.heating_recommendations) == 1 home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) # We predict a huge uplift but not quite as much as the EPC, due to some distinct differences between our # recommendation and the EPC assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 81.3 assert ending_epc['current-energy-efficiency'] == '92' # PATCH # We patch the simulation config, to reflect the ending EPC, to see if we get the ending EPC's config patched_simulation_config = { 'mainheat_energy_eff_ending': "Very Good", 'hot_water_energy_eff_ending': 'Good', 'has_boiler_ending': False, 'has_air_source_heat_pump_ending': True, 'has_electric_ending': True, 'has_lpg_ending': False, 'fuel_type_ending': 'electricity', 'switch_system_ending': 'programmer', 'no_control_ending': None, 'auxiliary_systems_ending': 'bypass', 'trvs_ending': 'trvs', "mainheatc_energy_eff_ending": 'Average' } # PATCHING property_recommendations_patch = Recommendations.insert_temp_recommendation_id( [recommender.heating_recommendations] ) property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations_patch, [] ) scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict_patch = model_api.predict_all( df=scoring_data_patch, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 88.9 # We still underpredict but the improvement is notable def test_offgrid(self): """ We test on a property we've worked with before, where we compare two options a) Upgrading to a boiler b) Upgrading to a heat pump :return: """ starting_epc = { 'low-energy-fixed-light-count': '', 'address': '6 Beech Road', 'uprn-source': 'Energy Assessor', 'floor-height': '2.4', 'heating-cost-potential': '612', 'unheated-corridor-length': '', 'hot-water-cost-potential': '123', 'construction-age-band': 'England and Wales: 1930-1949', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Good', 'environment-impact-potential': '87', 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '2278', 'address3': '', 'mainheatcont-description': 'Appliance thermostats', 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Dudley', 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '604', 'county': '', 'postcode': 'DY1 4BP', 'solar-water-heating-flag': 'N', 'constituency': 'E14000671', 'co2-emissions-potential': '1.0', 'number-heated-rooms': '4', 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '93', 'local-authority': 'E08000027', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2024-03-13', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '83', 'address1': '6 Beech Road', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Dudley North', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '60.0', 'building-reference-number': '10005780080', 'environment-impact-current': '41', 'co2-emissions-current': '5.0', 'roof-description': 'Pitched, 12 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'DUDLEY', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 67% of fixed outlets', 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '113', 'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2024-03-13 11:29:11', 'flat-top-storey': '', 'current-energy-rating': 'F', 'secondheat-description': 'None', 'walls-env-eff': 'Average', 'transaction-type': 'rental', 'uprn': '90055152', 'current-energy-efficiency': '32', 'energy-consumption-current': '491', 'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '113', 'lodgement-date': '2024-03-13', 'extension-count': '1', 'mainheatc-env-eff': 'Good', 'lmk-key': '78ddf851b660e599a0894924d0e6b503980f5e0ad1aa711f8411718dc2989c44', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '87', 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '67', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'Electric immersion, standard tariff' } cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = HeatingRecommender(property_instance=home) recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) recommender.recommend_boiler_upgrades(phase=0, system_change=True, exising_room_heaters=False) assert len(recommender.heating_recommendations) == 3 property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) # The ASHP isn't better under SAP, compared to a gas boiler with good heat controls assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [66.9, 65.5, 65.9]