import pytest from recommendations.SolarPvRecommendations import SolarPvRecommendations from backend.Property import Property from etl.epc.Record import EPCRecord import pandas as pd from datetime import datetime from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 from etl.solar.SolarPhotoSupply import SolarPhotoSupply from recommendations.Recommendations import Recommendations from backend.ml_models.api import ModelApi import msgpack class TestSolarPvRecommendations: @pytest.fixture def property_instance_invalid_type(self): # Setup the property_instance with an invalid property type epc_record = EPCRecord() epc_record.prepared_epc = { "property-type": "InvalidType", "county": "Broxbourne", "photo-supply": None } property_instance_invalid_type = Property(id=1, address="", postcode="", epc_record=epc_record) property_instance_invalid_type.roof = {"is_flat": False, "is_pitched": False, "is_roof_room": False} return property_instance_invalid_type @pytest.fixture def property_instance_invalid_roof(self): # Setup the property_instance with invalid roof type epc_record = EPCRecord() epc_record.prepared_epc = { "county": "Huntingdonshire", "property-type": "House", "photo-supply": None } property_instance_invalid_roof = Property(id=1, address="", postcode="", epc_record=epc_record) property_instance_invalid_roof.roof = {"is_flat": False, "is_pitched": False, "is_roof_room": False} return property_instance_invalid_roof @pytest.fixture def property_instance_has_solar_pv(self): # Setup the property_instance without existing solar pv epc_record = EPCRecord() epc_record.prepared_epc = {"photo-supply": "40", "county": "Huntingdonshire", "property-type": "House"} property_instance_has_solar_pv = Property(id=1, address="", postcode="", epc_record=epc_record) property_instance_has_solar_pv.roof = {"is_flat": True} return property_instance_has_solar_pv @pytest.fixture def property_instance_valid_all(self): # Setup a valid property_instance that passes all conditions epc_record = EPCRecord() epc_record.prepared_epc = {"property-type": "House", "photo-supply": None, "county": "Huntingdonshire"} property_instance_valid_all = Property(id=1, address="", postcode="", epc_record=epc_record) property_instance_valid_all.solar_pv_roof_area = 20 property_instance_valid_all.solar_pv_percentage = 40 property_instance_valid_all.roof = {"is_flat": True} return property_instance_valid_all def test_invalid_property_type(self, property_instance_invalid_type): solar_pv = SolarPvRecommendations(property_instance_invalid_type) solar_pv.recommend() assert not solar_pv.recommendation def test_invalid_roof_type(self, property_instance_invalid_roof): solar_pv = SolarPvRecommendations(property_instance_invalid_roof) solar_pv.recommend() assert not solar_pv.recommendation def test_existing_solar_pv(self, property_instance_has_solar_pv): solar_pv = SolarPvRecommendations(property_instance_has_solar_pv) solar_pv.recommend() assert not solar_pv.recommendation def test_valid_all_conditions(self, property_instance_valid_all): solar_pv = SolarPvRecommendations(property_instance_valid_all) solar_pv.recommend() assert solar_pv.recommendation == [ { 'parts': [], 'type': 'solar_pv', 'description': 'Install a 4 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on the roof', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 8527.0752, 'subtotal': 7105.896, 'vat': 1421.1791999999996, 'labour_hours': 72, 'labour_days': 2, 'photo_supply': 4000 } ] def test_model(self): """ This function tests the recommendation engine, in conjunction with the model :return: """ starting_epc = { 'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor', 'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '', 'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '85', 'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79', 'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N', 'constituency': 'E14000707', 'co2-emissions-potential': '1.5', 'number-heated-rooms': '5', 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '92', 'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-17', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '61', 'address1': '27 Cromwell Street', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430', 'environment-impact-current': '47', 'co2-emissions-current': '5.4', 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '72', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2021-12-01 10:12:23', 'flat-top-storey': '', 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '54', 'energy-consumption-current': '346', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '144', 'lodgement-date': '2021-12-01', 'extension-count': '2', 'mainheatc-env-eff': 'Good', 'lmk-key': '3ec5533af02ec78361c1f9bea8dd2e878c2c6fa6cf59e5cc505c3eeb038e0f91', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0', 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'hotwater-description': 'From main system' } ending_epc = { 'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor', 'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '', 'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '86', 'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79', 'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N', 'constituency': 'E14000707', 'co2-emissions-potential': '1.4', 'number-heated-rooms': '5', 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '84', 'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-12-21', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '49', 'address1': '27 Cromwell Street', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430', 'environment-impact-current': '55', 'co2-emissions-current': '4.4', 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '50.0', 'lighting-cost-potential': '72', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2021-12-21 17:33:09', 'flat-top-storey': '', 'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '65', 'energy-consumption-current': '277', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '144', 'lodgement-date': '2021-12-21', 'extension-count': '2', 'mainheatc-env-eff': 'Good', 'lmk-key': 'b0b19583c59afbc69db12f4d6c98cd8837e80da3214d577c426eb3e672d424fc', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '88', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0', 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'hotwater-description': 'From main system' } cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = SolarPvRecommendations(property_instance=home) recommender.recommend(phase=0) coverage_50_percent = [x for x in recommender.recommendation if x["photo_supply"] == 50] assert len(coverage_50_percent) == 2 property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_50_percent]) home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [65.9, 65.9] assert ending_epc["current-energy-efficiency"] == '65' def test_model2(self): data[["uprn", "sap_ending"]] # searcher = SearchEpc( address1="", postcode="", auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=", os_api_key="", full_address="", uprn=100030952942, ) searcher.find_property(False) ending_epc = { 'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', 'uprn-source': 'Energy Assessor', 'floor-height': '2.49', 'heating-cost-potential': '464', 'unheated-corridor-length': '', 'hot-water-cost-potential': '46', 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '91', 'glazed-type': 'not defined', 'heating-cost-current': '535', 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69', 'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N', 'constituency': 'E14000707', 'co2-emissions-potential': '0.7', 'number-heated-rooms': '3', 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '56', 'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical', 'inspection-date': '2022-08-24', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '18', 'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0', 'building-reference-number': '10002845316', 'environment-impact-current': '85', 'co2-emissions-current': '1.2', 'roof-description': 'Pitched, 300 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Average', 'photo-supply': '40.0', 'lighting-cost-potential': '65', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2022-08-24 15:39:42', 'flat-top-storey': '', 'current-energy-rating': 'B', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100030952942', 'current-energy-efficiency': '87', 'energy-consumption-current': '100', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '65', 'lodgement-date': '2022-08-24', 'extension-count': '0', 'mainheatc-env-eff': 'Good', 'lmk-key': 'e20be883431b1fed15db7fa1f52634fb7655d2b80c2fdad37df779f93ec4dafd', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '91', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } starting_epc = { 'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', 'uprn-source': 'Energy Assessor', 'floor-height': '2.49', 'heating-cost-potential': '464', 'unheated-corridor-length': '', 'hot-water-cost-potential': '46', 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '85', 'glazed-type': 'not defined', 'heating-cost-current': '535', 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69', 'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N', 'constituency': 'E14000707', 'co2-emissions-potential': '1.2', 'number-heated-rooms': '3', 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '102', 'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical', 'inspection-date': '2022-05-31', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40', 'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0', 'building-reference-number': '10002845316', 'environment-impact-current': '68', 'co2-emissions-current': '2.6', 'roof-description': 'Pitched, 300 mm loft insulation', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '65', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', 'lodgement-datetime': '2022-06-15 08:38:02', 'flat-top-storey': '', 'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100030952942', 'current-energy-efficiency': '68', 'energy-consumption-current': '227', 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '65', 'lodgement-date': '2022-06-15', 'extension-count': '0', 'mainheatc-env-eff': 'Good', 'lmk-key': 'ce181970b7077cb9b4626242bfb010b30a0e48541b5f22427e81f1adbeeec4f2', 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '85', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' } cleaning_data = read_dataframe_from_s3_parquet( bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", ) cleaned = read_from_s3( s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev" ) cleaned = msgpack.unpackb(cleaned, raw=False) photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") epc = EPCRecord( epc_records={ 'original_epc': starting_epc, 'full_sap_epc': {}, 'old_data': [] }, run_mode="newdata", cleaning_data=cleaning_data ) home = Property( id=0, address="", postcode="", epc_record=epc, already_installed={}, non_invasive_recommendations={}, ) home.in_conservation_area = False home.is_listed = False home.is_heritage = False home.restricted_measures = True home.get_components( cleaned=cleaned, photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds ) recommender = SolarPvRecommendations(property_instance=home) recommender.recommend(phase=0) coverage_40_percent = [x for x in recommender.recommendation if x["photo_supply"] == 40] assert len(coverage_40_percent) == 2 property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_40_percent]) home.create_base_difference_epc_record(cleaned_lookup=cleaned) home.adjust_difference_record_with_recommendations( property_recommendations, [] ) scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] ) model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) model_api.MODEL_PREFIXES = ["sap_change_predictions"] predictions_dict = model_api.predict_all( df=scoring_data, bucket="retrofit-data-dev", prediction_buckets={ "sap_change_predictions": "retrofit-sap-predictions-dev", } ) assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [87.1, 87.1] assert ending_epc["current-energy-efficiency"] == '87' assert starting_epc["current-energy-efficiency"] == '68'