From e43842d9803cb6d0d42927ce19a2473daeb5bfc1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 1 Aug 2024 22:07:19 +0100 Subject: [PATCH] handling the odd case of a double property --- backend/apis/GoogleSolarApi.py | 123 +++++++++++++++++- backend/app/plan/router.py | 13 ++ etl/bill_savings/data_collection.py | 2 + etl/energy_efficiency/app.py | 90 +++++++++++++ etl/xml_survey_extraction/app.py | 6 +- recommendations/HeatingControlRecommender.py | 6 +- recommendations/HeatingRecommender.py | 1 + recommendations/WallRecommendations.py | 20 ++- .../wall_energy_efficiency_values.py | 56 ++++++++ 9 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 etl/energy_efficiency/app.py create mode 100644 recommendations/wall_energy_efficiency_values.py diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 579e985d..c5167e32 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -9,6 +9,7 @@ from backend.app.db.functions.solar_functions import get_solar_data, store_batch from utils.logger import setup_logger from sklearn.preprocessing import MinMaxScaler from recommendations.Costs import Costs +from math import sin, cos, sqrt, atan2, radians logger = setup_logger() @@ -70,6 +71,9 @@ class GoogleSolarApi: # Indicates if we need to store the data to the db self.need_to_store = False + # Indicates if we think we have both units attached to a semi-detached property + self.double_property = False + def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None): """ Make an API request to retrieve building insights based on the given longitude and latitude, with retry @@ -116,7 +120,7 @@ class GoogleSolarApi: required_quality="MEDIUM", is_building=False, session=None, - uprn=None + uprn=None, ): """ Wrapper function that calls get_building_insights and extracts roof segments, with caching. @@ -147,6 +151,12 @@ class GoogleSolarApi: # Extract key data from the insights response self.roof_segments = self.insights_data["solarPotential"].get('roofSegmentStats', []) + # Automatically exclude north-facing segments + self.exclude_north_facing_segments() + # If a property is semi-detached, it's possible for us to include segments from an attached unit + if property_instance.data["built-form"] == "Semi-Detached": + self.exclude_likely_duplicate_surfaces() + self.roof_area = self.insights_data["solarPotential"]["wholeRoofStats"]['areaMeters2'] self.floor_area = self.insights_data["solarPotential"]["wholeRoofStats"]['groundAreaMeters2'] self.panel_area = ( @@ -162,9 +172,6 @@ class GoogleSolarApi: # It should be straightforward, but I'd rather see an actual instance of this happening raise NotImplementedError("Panel wattage is not 400W - implement me") - # Automatically exclude north-facing segments - self.exclude_north_facing_segments() - self.roof_segment_indexes = [segment['segmentIndex'] for segment in self.roof_segments] # We now start finding the solar panel configurations @@ -172,6 +179,11 @@ class GoogleSolarApi: energy_consumption=energy_consumption, is_building=is_building, property_instance=property_instance ) + # Finally, if we have a double property, we half the data we stored area + if self.double_property: + self.roof_area = self.roof_area / 2 + self.floor_area = self.floor_area / 2 + def save_to_db(self, session, uprns_to_location, scenario_type): if self.insights_data is None: raise ValueError("No api data to store") @@ -338,7 +350,13 @@ class GoogleSolarApi: # - surplus: this is the amount of additional energy generated, and therefore how much will be exported # - surplus_value: the value of the surplus energy - this feeds into generation_value, when relevant # - expected_payback_years: the number of years it will take to pay back the initial investment - lifetime_energy_consumption = energy_consumption * self.installation_life_span + + # If we have a double property (i.e. the solar api has returned data for two units) we size up the solar panels + # for double the consumption, as if for two units. + if self.double_property: + lifetime_energy_consumption = energy_consumption * 2 * self.installation_life_span + else: + lifetime_energy_consumption = energy_consumption * self.installation_life_span roi_results = [] for _, panel_config in panel_performance.iterrows(): lifetime_ac_kwh = panel_config["lifetime_ac_kwh"] @@ -408,6 +426,31 @@ class GoogleSolarApi: panel_performance["expected_payback_years"] = np.ceil(panel_performance["expected_payback_years"]).astype(int) + if self.double_property: + # Now that we've optimise to an energy consumption that is double the original, we need to half the + # results + panel_performance["n_panels_halved"] = panel_performance["n_panels"] / 2 + n_panels_required = {int(x) for x in np.floor(panel_performance["n_panels"] / 2)} + # We filter the data on this number of panels + panel_performance = panel_performance[panel_performance["n_panels_halved"].isin(n_panels_required)] + # We half the generation values + for col in [ + "yearly_dc_energy", + "total_cost", + "panneled_roof_area", + "array_wattage", + "initial_ac_kwh_per_year", + "lifetime_ac_kwh", + "lifetime_dc_kwh", + "generation_value", + "generation_deficit", + "surplus" + ]: + panel_performance[col] = panel_performance[col] / 2 + + panel_performance["n_panels"] = panel_performance["n_panels_halved"] + panel_performance = panel_performance.drop(columns=["n_panels_halved"]) + self.panel_performance = panel_performance def exclude_north_facing_segments(self): @@ -427,3 +470,73 @@ class GoogleSolarApi: filtered_segments.append(segment) self.roof_segments = filtered_segments + + @staticmethod + def haversine(lat1, lon1, lat2, lon2): + """ + Calculate the great-circle distance between two points on the Earth + given their latitude and longitude in decimal degrees. Using haversine formula. + """ + R = 6373.0 # approximate radius of earth in km + + lat1 = radians(lat1) + lon1 = radians(lon1) + lat2 = radians(lat2) + lon2 = radians(lon2) + + dlon = lon2 - lon1 + dlat = lat2 - lat1 + + a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2 + c = 2 * atan2(sqrt(a), sqrt(1 - a)) + + distance = R * c + return distance + + def exclude_likely_duplicate_surfaces(self): + """ + By checking the azimuth of the segments, we can exclude any segments that are likely to be duplicates + :return: + """ + + def is_similar(segment1, segment2, azimuth_tol=20): + azimuth_diff = abs(segment1['azimuthDegrees'] - segment2['azimuthDegrees']) + return azimuth_diff <= azimuth_tol + + property_center = self.insights_data["center"] + + deduped_segments = [] + for segment in self.roof_segments: + if not deduped_segments: + deduped_segments.append(segment) + continue + + similar_segments = [s for s in deduped_segments if is_similar(segment, s)] + if not similar_segments: + deduped_segments.append(segment) + else: + # Compare distances to the property center and keep the closer segment + for similar_segment in similar_segments: + current_dist = self.haversine( + property_center['latitude'], property_center['longitude'], + segment['center']['latitude'], segment['center']['longitude'] + ) + similar_dist = self.haversine( + property_center['latitude'], property_center['longitude'], + similar_segment['center']['latitude'], similar_segment['center']['longitude'] + ) + + if current_dist < similar_dist: + deduped_segments.remove(similar_segment) + deduped_segments.append(segment) + + # If we have a semi-detached property that has duplicated segments, we should expect to half the number of + # segments + if len(deduped_segments) < len(self.roof_segments): + if len(deduped_segments) != len(self.roof_segments) / 2: + raise ValueError("We don't have half the number of segments that we started with") + + # Because the segments are duplicated, but the sizes aren't necessarily split perfectly in half, what + # we need to do is perform the solar analysis and then half the results. We set an indicator which + # implies we should do this + self.double_property = True diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index b6175153..a108176b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -439,6 +439,8 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Performing solar analysis") # TODO: Tidy this up + # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour + # building_ids = [ { "building_id": p.building_id, @@ -709,6 +711,17 @@ async def trigger_plan(body: PlanTriggerRequest): ] recommendations[property_id] = final_recommendations + # df = [] + # for rec in recommendations[list(recommendations.keys())[0]]: + # df.append( + # { + # "id": rec["recommendation_id"], + # "description": rec["description"], + # "sap": rec["sap_points"], + # } + # ) + # df = pd.DataFrame(df) + # 1) the property data # 2) the property details (epc) # 3) the recommendations diff --git a/etl/bill_savings/data_collection.py b/etl/bill_savings/data_collection.py index 6cc2d581..15a52663 100644 --- a/etl/bill_savings/data_collection.py +++ b/etl/bill_savings/data_collection.py @@ -131,7 +131,9 @@ def app(): sample_size = 500 energy_consumption_data = [] + cavity_walls_data = [] for i, directory in tqdm(enumerate(epc_directories), total=len(epc_directories)): + # Skip the first 50 # if i < 57: # continue diff --git a/etl/energy_efficiency/app.py b/etl/energy_efficiency/app.py new file mode 100644 index 00000000..23f9d33f --- /dev/null +++ b/etl/energy_efficiency/app.py @@ -0,0 +1,90 @@ +import inspect +import pandas as pd +from tqdm import tqdm +from pathlib import Path + +src_file_path = inspect.getfile(lambda: None) + +EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" + + +def app(): + # For EPCs lodged from 2020 onwards, this collects data on the energy efficiency categories for wall insulation + # so that when we simulate, we know what the resulting energy efficiency category will be + + epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] + + date_cutoff = "2020-01-01" + walls_data = [] + ashp_data = [] + for i, directory in tqdm(enumerate(epc_directories), total=len(epc_directories)): + data = pd.read_csv(directory / "certificates.csv", low_memory=False) + # Rename the columns to the same format as the api returns + data.columns = [c.replace("_", "-").lower() for c in data.columns] + + insulated_walls = data[ + data["walls-description"].isin( + [ + "Cavity wall, filled cavity", + "Solid brick, with internal insulation", + "Solid brick, with external insulation", + ] + ) + ] + insulated_walls = insulated_walls[~pd.isnull(insulated_walls["uprn"])] + insulated_walls = insulated_walls[ + pd.to_datetime(insulated_walls["lodgement-date"]) >= date_cutoff + ] + + ashp = data[ + data["mainheat-description"] == "Air source heat pump, radiators, electric" + ] + ashp = ashp[~pd.isnull(ashp["uprn"])] + ashp = ashp[ + pd.to_datetime(ashp["lodgement-date"]) >= date_cutoff + ] + + walls_data.append(insulated_walls) + ashp_data.append(ashp) + + walls_df = pd.concat(walls_data) + ashp_df = pd.concat(ashp_data) + + ashp_agg = ( + ashp_df. + groupby( + ["construction-age-band", "mainheat-description", "mainheatcont-description", "mainheat-energy-eff", + "mainheatc-energy-eff"] + ) + .size() + .reset_index() + ) + ashp_agg = ashp_agg[ + ashp_agg["mainheatcont-description"].isin( + ["Programmer, TRVs and bypass", "Time and temperature zone control"] + ) + ] + + aggregations = {} + for description in [ + "Cavity wall, filled cavity", "Solid brick, with internal insulation", "Solid brick, with external insulation" + ]: + aggregation = walls_df[ + walls_df["walls-description"] == description + ].groupby( + ["construction-age-band", "walls-energy-eff"] + ).size().reset_index().rename(columns={0: "count"}) + + # For each grouping of age band, we use the most populus energy efficiency category + aggregation_deduped = aggregation.sort_values("count", ascending=False).drop_duplicates("construction-age-band") + aggregations[description] = aggregation_deduped + + # Since these tables are small, we just convert them to python dictionaries + # This data is just held in the wall_energy_efficiency_values script, rather than s3 + df1 = aggregations["Cavity wall, filled cavity"] + df2 = aggregations["Solid brick, with internal insulation"] + df3 = aggregations["Solid brick, with external insulation"] + + df1.to_dict("records") + df2.to_dict("records") + df3.to_dict("records") diff --git a/etl/xml_survey_extraction/app.py b/etl/xml_survey_extraction/app.py index a8bffc73..92451d76 100644 --- a/etl/xml_survey_extraction/app.py +++ b/etl/xml_survey_extraction/app.py @@ -27,7 +27,7 @@ SCENARIOS = { "already_installed_file_path": "", "patches_file_path": "", "non_invasive_recommendations_file_path": "", - "exclusions": ["floor_insulation", "fireplace", "solar_pv", "heating"], + "exclusions": ["floor_insulation", "fireplace", "solar_pv", "heating", 'lighting'], "budget": None, "scenario_name": "Low Hanging Fruit", "multi_plan": True, @@ -42,7 +42,7 @@ SCENARIOS = { "already_installed_file_path": "", "patches_file_path": "", "non_invasive_recommendations_file_path": "", - "exclusions": ["floor_insulation", "fireplace"], + "exclusions": ["floor_insulation", "fireplace", 'lighting'], "budget": None, "scenario_name": "Deep Retrofit", "multi_plan": True, @@ -57,7 +57,7 @@ SCENARIOS = { "already_installed_file_path": "", "patches_file_path": "", "non_invasive_recommendations_file_path": "", - "exclusions": ["fireplace"], + "exclusions": ["fireplace", 'lighting'], "budget": None, "scenario_name": "Whole House Retrofit", "multi_plan": True, diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index ef0df011..6e827084 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -291,7 +291,11 @@ class HeatingControlRecommender: simulation_config = check_simulation_difference( new_config=ending_config, old_config=self.property.main_heating_controls ) - simulation_config["mainheatc_energy_eff_ending"] = "Average" + # Only adjust if the current system is below good + if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor"]: + simulation_config["mainheatc_energy_eff_ending"] = "Average" + else: + simulation_config["mainheatc_energy_eff_ending"] = self.property.data["mainheatc-energy-eff"] description_simulation = { "mainheatcont-description": new_controls_description, diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index ab377369..4d91f21b 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -257,6 +257,7 @@ class HeatingRecommender: f" The cost includes the £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant" ) + print("TEMP UPDATED FOR 77 Perryn!!!!!") simulation_config = { "mainheat_energy_eff_ending": "Good", "hot_water_energy_eff_ending": "Good" diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 448b34e8..4ef747f7 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -13,6 +13,7 @@ from recommendations.recommendation_utils import ( ) from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION from recommendations.Costs import Costs +from recommendations.wall_energy_efficiency_values import cavity_wall_energy_eff, iwi_energy_eff, ewi_energy_eff from utils.logger import setup_logger logger = setup_logger() @@ -404,11 +405,28 @@ class WallRecommendations(Definitions): simulation_config = {} if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]: + if wall_ending_config["is_cavity_wall"]: + efficiency_data = [ + x for x in cavity_wall_energy_eff if + x["construction-age-band"] == self.property.construction_age_band + ][0] + elif wall_ending_config["internal_insulation"]: + efficiency_data = [ + x for x in iwi_energy_eff if + x["construction-age-band"] == self.property.construction_age_band + ][0] + else: + efficiency_data = [ + x for x in ewi_energy_eff if + x["construction-age-band"] == self.property.construction_age_band + ][0] + simulation_config = { - "walls_energy_eff_ending": "Good" + "walls_energy_eff_ending": efficiency_data["walls-energy-eff"] } # We check if we have double insulation in any instances + # TODO: We should pull the energy efficiency categories on double insulation instances, though it's quite rate double_insulation = ( (wall_ending_config["is_filled_cavity"] and wall_ending_config["external_insulation"]) or (wall_ending_config["is_filled_cavity"] and wall_ending_config["internal_insulation"]) or diff --git a/recommendations/wall_energy_efficiency_values.py b/recommendations/wall_energy_efficiency_values.py new file mode 100644 index 00000000..bfd43eb2 --- /dev/null +++ b/recommendations/wall_energy_efficiency_values.py @@ -0,0 +1,56 @@ +cavity_wall_energy_eff = [ + {'construction-age-band': 'England and Wales: 1950-1966', 'walls-energy-eff': 'Average', 'count': 605820}, + {'construction-age-band': 'England and Wales: 1967-1975', 'walls-energy-eff': 'Average', 'count': 410998}, + {'construction-age-band': 'England and Wales: 1930-1949', 'walls-energy-eff': 'Average', 'count': 263575}, + {'construction-age-band': 'England and Wales: 1976-1982', 'walls-energy-eff': 'Good', 'count': 206654}, + {'construction-age-band': 'England and Wales: 1983-1990', 'walls-energy-eff': 'Good', 'count': 106489}, + {'construction-age-band': 'England and Wales: 1900-1929', 'walls-energy-eff': 'Average', 'count': 58399}, + {'construction-age-band': 'England and Wales: 1991-1995', 'walls-energy-eff': 'Good', 'count': 58252}, + {'construction-age-band': 'England and Wales: 1996-2002', 'walls-energy-eff': 'Good', 'count': 35141}, + {'construction-age-band': 'England and Wales: 2003-2006', 'walls-energy-eff': 'Good', 'count': 7194}, + {'construction-age-band': 'England and Wales: 2007-2011', 'walls-energy-eff': 'Good', 'count': 2639}, + {'construction-age-band': 'England and Wales: before 1900', 'walls-energy-eff': 'Average', 'count': 2495}, + {'construction-age-band': 'England and Wales: 2012 onwards', 'walls-energy-eff': 'Very Good', 'count': 1158}, + {'construction-age-band': 'England and Wales: 2007 onwards', 'walls-energy-eff': 'Good', 'count': 357}, + {'construction-age-band': 'INVALID!', 'walls-energy-eff': 'Very Good', 'count': 88} +] + +iwi_energy_eff = [ + {'construction-age-band': 'England and Wales: 1900-1929', 'walls-energy-eff': 'Good', 'count': 22415}, + {'construction-age-band': 'England and Wales: before 1900', 'walls-energy-eff': 'Good', + 'count': 13422}, + {'construction-age-band': 'England and Wales: 1930-1949', 'walls-energy-eff': 'Good', 'count': 6640}, + {'construction-age-band': 'England and Wales: 1950-1966', 'walls-energy-eff': 'Good', 'count': 1391}, + {'construction-age-band': 'England and Wales: 1967-1975', 'walls-energy-eff': 'Good', 'count': 663}, + {'construction-age-band': 'England and Wales: 2003-2006', 'walls-energy-eff': 'Very Good', + 'count': 516}, + {'construction-age-band': 'England and Wales: 2007-2011', 'walls-energy-eff': 'Very Good', + 'count': 463}, + {'construction-age-band': 'England and Wales: 2012 onwards', 'walls-energy-eff': 'Very Good', + 'count': 353}, + {'construction-age-band': 'England and Wales: 1996-2002', 'walls-energy-eff': 'Good', 'count': 218}, + {'construction-age-band': 'England and Wales: 1983-1990', 'walls-energy-eff': 'Very Good', + 'count': 166}, + {'construction-age-band': 'England and Wales: 1976-1982', 'walls-energy-eff': 'Very Good', + 'count': 121}, + {'construction-age-band': 'England and Wales: 1991-1995', 'walls-energy-eff': 'Good', 'count': 104}, + {'construction-age-band': 'England and Wales: 2007 onwards', 'walls-energy-eff': 'Very Good', + 'count': 74}, {'construction-age-band': 'INVALID!', 'walls-energy-eff': 'Very Good', 'count': 26} +] + +ewi_energy_eff = [ + {'construction-age-band': 'England and Wales: 1900-1929', 'walls-energy-eff': 'Good', 'count': 18427}, + {'construction-age-band': 'England and Wales: 1930-1949', 'walls-energy-eff': 'Good', 'count': 17803}, + {'construction-age-band': 'England and Wales: 1950-1966', 'walls-energy-eff': 'Good', 'count': 4306}, + {'construction-age-band': 'England and Wales: before 1900', 'walls-energy-eff': 'Good', 'count': 2955}, + {'construction-age-band': 'England and Wales: 1967-1975', 'walls-energy-eff': 'Good', 'count': 647}, + {'construction-age-band': 'England and Wales: 1976-1982', 'walls-energy-eff': 'Very Good', 'count': 188}, + {'construction-age-band': 'England and Wales: 2007-2011', 'walls-energy-eff': 'Very Good', 'count': 73}, + {'construction-age-band': 'England and Wales: 2003-2006', 'walls-energy-eff': 'Very Good', 'count': 49}, + {'construction-age-band': 'England and Wales: 2012 onwards', 'walls-energy-eff': 'Very Good', 'count': 37}, + {'construction-age-band': 'England and Wales: 1983-1990', 'walls-energy-eff': 'Good', 'count': 31}, + {'construction-age-band': 'England and Wales: 1996-2002', 'walls-energy-eff': 'Very Good', 'count': 21}, + {'construction-age-band': 'England and Wales: 1991-1995', 'walls-energy-eff': 'Good', 'count': 14}, + {'construction-age-band': 'England and Wales: 2007 onwards', 'walls-energy-eff': 'Very Good', 'count': 8}, + {'construction-age-band': 'INVALID!', 'walls-energy-eff': 'Very Good', 'count': 4} +]