diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index cc2ee4a9..44178792 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -709,8 +709,13 @@ class SearchEpc: self.full_sap_epc = {} # Finally, set a standardised address 1 and postcode - self.address_clean = self.ordnance_survey_client.address_os - self.postcode_clean = self.ordnance_survey_client.postcode_os + self.address_clean = ( + self.ordnance_survey_client.address_os if self.ordnance_survey_client.address_os else self.address1 + ) + self.postcode_clean = ( + self.ordnance_survey_client.postcode_os if self.ordnance_survey_client.postcode_os else + self.postcode + ) return os_response = self.ordnance_survey_client.get_places_api() diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 9854abe8..ebaf482d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -52,6 +52,10 @@ def patch_epc(patch, epc_records): """ for patch_variable, patch_value in patch.items(): + + if patch_variable in ["address", "postcode"]: + continue + if patch_value == "": continue if patch_variable in epc_records["original_epc"]: @@ -268,23 +272,26 @@ async def trigger_plan(body: PlanTriggerRequest): postcode=config["postcode"], uprn=uprn, auth_token=get_settings().EPC_AUTH_TOKEN, - os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY + os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY, ) - epc_searcher.find_property() + epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None) + epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None) + # For the moment, our OS API access is unavailable, so we skip and interpolate + epc_searcher.find_property(skip_os=True) # Create a record in db property_id, is_new = create_property( session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn ) - if not is_new: - continue - - create_property_targets( - session, - property_id=property_id, - portfolio_id=body.portfolio_id, - epc_target=body.goal_value, - heat_demand_target=None - ) + # if not is_new: + # continue + # + # create_property_targets( + # session, + # property_id=property_id, + # portfolio_id=body.portfolio_id, + # epc_target=body.goal_value, + # heat_demand_target=None + # ) epc_records = { 'original_epc': epc_searcher.newest_epc.copy(), @@ -373,6 +380,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Preparing data for scoring in sap change api") recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) + recommendations_scoring_data = recommendations_scoring_data.drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 251c016a..39ea5a98 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -63,6 +63,14 @@ class PropertyValuation: 90093693: 279_000, # Based on Zoopla 90055152: 149_000, # Based on Zoopla 90028499: 238_000, # Based on Zoopla + # IMMO Dudley Pilot 2- search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/ + 90039318: 177_000, # Based on Zoopla + 90038384: 170_000, # Based on Zoopla + 90105380: 185_000, # Based on Zoopla + 90124001: 165_000, # Based on Zoopla + 90013980: 148_000, # Based on Zoopla + 90087154: 184_000, # Based on Zoopla + 90046817: 167_000, # Based on Zoopla } # We base our valuation uplifts on a number of sources diff --git a/etl/customers/gla_croydon_demo/asset_list.py b/etl/customers/gla_croydon_demo/asset_list.py index 7dde8926..1655979b 100644 --- a/etl/customers/gla_croydon_demo/asset_list.py +++ b/etl/customers/gla_croydon_demo/asset_list.py @@ -34,8 +34,9 @@ def app(): low_memory=False ) - z = epc_data.groupby(["WALLS_DESCRIPTION", "WALLS_ENERGY_EFF"]).size().reset_index(name="count") - z = z[z["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"] + z = epc_data[epc_data["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"] + z["HOTWATER_DESCRIPTION"].value_counts() + z["MAIN_FUEL"].value_counts() # Filter on entries where we have a UPRN epc_data = epc_data[~pd.isnull(epc_data["UPRN"])] diff --git a/etl/customers/immo/pilot/asset_list_2.py b/etl/customers/immo/pilot/asset_list_2.py new file mode 100644 index 00000000..52260f57 --- /dev/null +++ b/etl/customers/immo/pilot/asset_list_2.py @@ -0,0 +1,152 @@ +import pandas as pd +from utils.s3 import read_excel_from_s3 +from utils.s3 import save_csv_to_s3 + +USER_ID = 8 +PORTFOLIO_ID = 72 + +# For +patches = [ + { + 'address': '116 Parkes Hall Road', + 'postcode': 'DY1 3RJ', + 'uprn': '90046817', + 'walls-description': 'Cavity wall, filled cavity', + 'walls-energy-eff': 'Average', + 'roof-description': 'Pitched, 270 mm loft insulation', + 'roof-energy-eff': 'Good', + 'windows-description': 'Fully double glazed', + 'windows-energy-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, mains gas', + 'mainheat-energy-eff': 'Good', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'mainheatc-energy-eff': 'Good', + 'lighting-description': 'Low energy lighting in 27% of fixed outlets', + 'lighting-energy-eff': 'Average', + 'floor-description': 'Solid, no insulation (assumed)', + 'secondheat-description': 'None', + 'current-energy-efficiency': '73', + 'current-energy-rating': 'C', + 'energy-consumption-current': '184', + 'co2-emissions-current': '2.4', + 'potential-energy-efficiency': '88', + 'total-floor-area': '73', + 'construction-age-band': 'England and Wales: 1930-1949', + 'property-type': 'House', + 'built-form': 'Mid-Terrace', + } +] + +# This is information that is found as a result of the non-invasives, that mean that certain measures +# have been installed already. To reflect this in the front end, it is included in the recommendation, however +# the cost is removed and instead, a message is presented saying that the measure is already installed. +already_installed = [ + { + 'address': '28 Sangwin Road', 'postcode': 'WV14 9EQ', "already_installed": ["loft_insulation"] + }, + { + 'address': '51 Hillwood Road', 'postcode': 'B62 8NQ', "already_installed": ["loft_insulation"] + }, + { + 'address': '47 Watsons Close', 'postcode': 'DY2 7HL', "already_installed": ["loft_insulation"] + }, + { + 'address': '44 Hatfield Road', + 'postcode': 'DY9 7LW', + "already_installed": ["loft_insulation", "cavity_wall_insulation"] + } +] + +non_invasive_recommendations = [] + + +def app(): + raw_asset_list = read_excel_from_s3( + bucket_name="retrofit-datalake-dev", + file_key="customers/Immo/Dudley Asset List - Hestia - pilot2.xlsx", + header_row=0 + ) + + raw_asset_list = raw_asset_list[raw_asset_list["in_pilot"]].copy() + + # Extract address and postcode + raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0] + raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip() + + # We're provided with number of bathrooms and number of bedrooms. + # THe UPRNs are not the official ones + asset_list = raw_asset_list.rename( + columns={ + "No. of Beds": "n_bedrooms", + "No. of WC's": "n_bathrooms", + 'Property Type': 'property_type', + 'Architype': 'built_form' + } + ) + + # Remap the values + asset_list["built_form"] = asset_list["built_form"].map({ + "SEMI DETACHED": "Semi-Detached", + "MID TERRACE": "Mid-Terrace", + "END TERRACE": "End-Terrace", + }) + + # Store the asset list in s3 + filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + # Store overrides in s3 + already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json" + save_csv_to_s3( + dataframe=pd.DataFrame(already_installed), + bucket_name="retrofit-plan-inputs-dev", + file_name=already_installed_filename + ) + + # Store patches in s3 + patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json" + save_csv_to_s3( + dataframe=pd.DataFrame(patches), + bucket_name="retrofit-plan-inputs-dev", + file_name=patches_filename + ) + + # Store non-invasive recommendations in S3 + non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json" + save_csv_to_s3( + dataframe=pd.DataFrame(non_invasive_recommendations), + bucket_name="retrofit-plan-inputs-dev", + file_name=non_invasive_recommendations_filename + ) + + # EPC C portoflio + body = { + "portfolio_id": str(PORTFOLIO_ID), + "housing_type": "Private", + "goal": "Increase EPC", + "goal_value": "C", + "trigger_file_path": filename, + "already_installed_file_path": already_installed_filename, + "patches_file_path": patches_filename, + "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, + "budget": None, + } + print(body) + + # EPC B portoflio + body = { + "portfolio_id": str(PORTFOLIO_ID + 1), + "housing_type": "Private", + "goal": "Increase EPC", + "goal_value": "B", + "trigger_file_path": filename, + "already_installed_file_path": already_installed_filename, + "patches_file_path": patches_filename, + "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, + "budget": None, + } + print(body) diff --git a/etl/epc/Record.py b/etl/epc/Record.py index e74330a2..9a965c6a 100644 --- a/etl/epc/Record.py +++ b/etl/epc/Record.py @@ -191,7 +191,7 @@ class EPCRecord: This method will clean the records using the data processor """ epc_data_processor = EPCDataProcessor( - data=self.epc_record_as_dataframe("prepared_epc"), + data=self.epc_record_as_dataframe("prepared_epc").copy(), run_mode="newdata", cleaning_averages=self.cleaning_data, ) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 852bb11f..d7a8ad2f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -67,18 +67,12 @@ LOW_CARBON_COMBI_BOILER = 2200 # https://www.greenmatch.co.uk/boilers/35kw-boiler # https://www.greenmatch.co.uk/boilers/40kw-boiler # These are exclusive of installation costs -COMBI_BOILER_COSTS = { +CONDENSING_BOILER_COSTS = { "30kw": 1550, "35kw": 1610, "40kw": 1625 } -CONVENTIONAL_BOILER_COSTS = { - "30kw": 1117, - "35kw": 1546, - "40kw": 1776 -} - # Assumes 3 hours to remove each heater (including re-decorating) ROOM_HEATER_REMOVAL_COST = 120 ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3 @@ -1179,7 +1173,7 @@ class Costs: estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators) return round(estimated_radiators) - def boiler(self, is_combi, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms): + def boiler(self, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms): """ Based on a basic estimate of median value £2600 to install a low carbon combi boiler First time central heating vosts can als be found here: @@ -1187,7 +1181,7 @@ class Costs: :return: """ - unit_cost = COMBI_BOILER_COSTS[size] if is_combi else CONVENTIONAL_BOILER_COSTS[size] + unit_cost = CONDENSING_BOILER_COSTS[size] # The unit cost is the cost without VAT # We now need to estimate the cost of the works labour_days = 2 diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 432dc6a6..537125a1 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -15,7 +15,8 @@ class HeatingRecommender: self.property = property_instance self.costs = Costs(self.property) - self.recommendations = [] + self.heating_recommendations = [] + self.heating_control_recommendations = [] def recommend(self, phase=0): @@ -23,7 +24,8 @@ class HeatingRecommender: # the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this # in the Costs class, stored as SYSTEM_FLUSH_COST - self.recommendations = [] + self.heating_recommendations = [] + self.heating_control_recommendations = [] # This first iteration of the recommender will provide very basic recommendation # We recommend heating controls based on the main heating system @@ -254,7 +256,7 @@ class HeatingRecommender: system_change=system_change ) - self.recommendations.extend(recommendations) + self.heating_recommendations.extend(recommendations) @staticmethod def estimate_boiler_size(property_type, built_form, floor_area, floor_height, num_heated_rooms): @@ -312,7 +314,15 @@ class HeatingRecommender: simulation_config = {} boiler_costs = {} boiler_recommendation = {} - if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]: + + has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] + + has_inefficient_mains_water = ( + self.property.hotwater["clean_description"] in ["From main system"] and + self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] + ) + + if has_inefficient_space_heating or has_inefficient_mains_water: boiler_size = self.estimate_boiler_size( property_type=self.property.data["property-type"], built_form=self.property.data["built-form"], @@ -321,22 +331,12 @@ class HeatingRecommender: num_heated_rooms=self.property.data["number-heated-rooms"], ) - # We recommend a combi boiler under the following conditions - # 1) If there are 4 or fewer rooms (we don't use heqted rooms because none of the rooms could be - # heated if there is no existing heating system). - # 2) There 1 or fewer bathrooms - # Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple - # bathrooms - is_combi = ( - (self.property.number_of_rooms <= 4) and - (self.property.n_bathrooms in [None, 0, 1]) - ) - if is_combi: - description = "Upgrade to a new combi boiler" - else: - description = "Upgrade to a new gas condensing boiler" + description = "Upgrade to a new condensing boiler" - simulation_config = {"mainheat_energy_eff_ending": "Good"} + simulation_config = { + "mainheat_energy_eff_ending": "Good", + "hot_water_energy_eff_ending": "Good" + } if system_change: # Installation of a boiler improves the hot water system so we need to reflect this in # the outcome of the recommendation @@ -363,7 +363,6 @@ class HeatingRecommender: } boiler_costs = self.costs.boiler( - is_combi=is_combi, size=f"{boiler_size}kw", exising_room_heaters=exising_room_heaters, system_change=system_change, @@ -397,9 +396,13 @@ class HeatingRecommender: controls_recommender.recommend(heating_description="Boiler and radiators, mains gas") # We may have 2 recommendations from the heating controls - if not controls_recommender.recommendation: + if not controls_recommender.recommendation and not boiler_recommendation: return + if not system_change and len(boiler_recommendation): + # If there is not a system change, we add the boiler recommendation at point. + self.heating_recommendations.extend([boiler_recommendation]) + if system_change: # We combine the heating and controls recommendations, in the case of a system change combined_recommendations = [] @@ -416,12 +419,12 @@ class HeatingRecommender: combined_recommendations.extend(combined_recommendation) # Overwrite the existing boiler recommendation - self.recommendations.extend(combined_recommendations) + self.heating_recommendations.extend(combined_recommendations) else: # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade # but we'll only upgrade if we have a heating recommendation has_heating_recommendation = any( - recommendation["type"] == "heating" for recommendation in self.recommendations + rec["type"] == "heating" for rec in self.heating_recommendations ) if has_heating_recommendation: recommendation_phase += 1 @@ -430,6 +433,6 @@ class HeatingRecommender: for recommendation in controls_recommender.recommendation: recommendation["phase"] = recommendation_phase - self.recommendations.extend(controls_recommender.recommendation) + self.heating_control_recommendations.extend(controls_recommender.recommendation) return diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 5960d7be..06dc2d61 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -110,12 +110,24 @@ class Recommendations: # Heating and Electical systems if "heating" not in self.exclusions: self.heating_recommender.recommend(phase=phase) - if self.heating_recommender.recommendations: - property_recommendations.append(self.heating_recommender.recommendations) + if ( + self.heating_recommender.heating_recommendations or + self.heating_recommender.heating_control_recommendations + ): + if self.heating_recommender.heating_recommendations: + property_recommendations.append(self.heating_recommender.heating_recommendations) + + if self.heating_recommender.heating_control_recommendations: + property_recommendations.append(self.heating_recommender.heating_control_recommendations) + # We check if we have distinct heating and heating controls recommendations # If so, we increment by 2 (one of the heating system, one for the heating controls) # otherwise we incremenet by 1 - max_used_phase = max([rec["phase"] for rec in self.heating_recommender.recommendations]) + max_used_phase = max( + [rec["phase"] for rec in + self.heating_recommender.heating_recommendations + + self.heating_recommender.heating_control_recommendations] + ) amount_to_increment = max_used_phase - phase + 1 phase += amount_to_increment diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 58cf9735..b44557ab 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -56,14 +56,18 @@ class SolarPvRecommendations: if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv: return + solar_pv_percentage = self.property.solar_pv_percentage + # We round up to the neaest 10% + solar_pv_percentage = np.ceil(solar_pv_percentage * 10) / 10 + # For the solar recommendations, we produce the following scenarios: # 1) Solar panels only, we present a high, medium and low coverage # 2) With and without battery roof_coverage_scenarios = [ - self.property.solar_pv_percentage - 0.1, self.property.solar_pv_percentage, + solar_pv_percentage - 0.1, solar_pv_percentage, ] - if self.property.solar_pv_percentage <= 0.4: - roof_coverage_scenarios.append(self.property.solar_pv_percentage + 0.1) + if solar_pv_percentage <= 0.4: + roof_coverage_scenarios.append(solar_pv_percentage + 0.1) # We make sure we haven't gone too low or high - we allow no more than 60% coverage roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 0.6] # If we only have two scenarios, we add a coverage scenario 10% less than the smallest