From c5b7775a2342f0a09e7acb49168472fe3a377611 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 16:27:17 +0100 Subject: [PATCH 01/29] created hashing of uprn --- backend/SearchEpc.py | 3 +- etl/customers/eon/pilot_asset_list.py | 54 +++++++++++++-------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index b5ec8c46..e7e717ac 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -405,7 +405,8 @@ class SearchEpc: else: raise ValueError("Multiple UPRNs found - investigate me") - uprn = uprns.pop() if uprns else None + # If we do not have a UPRN, we create one based on a hash of the address & postcoce + uprn = uprns.pop() if uprns else hash(self.address1 + self.postcode) if self.fast: return newest_epc, [], {}, "", "", None diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index 05e459cb..298b34ba 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -170,39 +170,39 @@ def app(): # For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free # trial with Ordnance Survey with my personal account as a temporary solution. # Let's just pull the full EPC data for this - asset_list_with_uprn = [] - for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): - if row <= 104: - continue - time.sleep(1.1) - searcher = SearchEpc( - address1=property_meta["address"], - postcode=property_meta["postcode"], - auth_token=EPC_AUTH_TOKEN, - os_api_key=ORDNANCE_SURVEY_API_KEY, - full_address=", ".join([property_meta["address"], property_meta["postcode"]]) - ) - - # Let's just find the UPRN - searcher.ordnance_survey_client.get_places_api() - - uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] - - # searcher.find_property(skip_os=False) - - asset_list_with_uprn.append( - { - **property_meta, - "uprn": uprn, - } - ) + # asset_list_with_uprn = [] + # for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): + # if row <= 104: + # continue + # time.sleep(1.1) + # searcher = SearchEpc( + # address1=property_meta["address"], + # postcode=property_meta["postcode"], + # auth_token=EPC_AUTH_TOKEN, + # os_api_key=ORDNANCE_SURVEY_API_KEY, + # full_address=", ".join([property_meta["address"], property_meta["postcode"]]) + # ) + # + # # Let's just find the UPRN + # searcher.ordnance_survey_client.get_places_api() + # + # uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] + # + # # searcher.find_property(skip_os=False) + # + # asset_list_with_uprn.append( + # { + # **property_meta, + # "uprn": uprn, + # } + # ) # Store this as a backup # import pandas as pd # asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False) # Read in - # asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") + asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") # Store the asset list and create the portfolio payload asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) From fc1468efdac13a9331f64d7785917a7759b46b0c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 16:31:18 +0100 Subject: [PATCH 02/29] enforcing datetime is string --- etl/bill_savings/KwhData.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etl/bill_savings/KwhData.py b/etl/bill_savings/KwhData.py index 6b5f594a..24ce9f2c 100644 --- a/etl/bill_savings/KwhData.py +++ b/etl/bill_savings/KwhData.py @@ -259,6 +259,9 @@ class KwhData: # Create new features: data['estimate_annual_kwh'] = data['energy-consumption-current'] * data['total-floor-area'] + # Ensure this is string, because we could have mixed types + data["lodgement-datetime"] = data["lodgement-datetime"].astype(str) + if save: self.model_training_data_filepath = f"energy_consumption/{self.run_date}/training_data.parquet" logger.info(f"Storing energy consumption dataset in s3 at {self.consumption_data_filepath}") From 6029aad07c1761dbcfaa1cb7231092f735f641f8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 19:32:46 +0100 Subject: [PATCH 03/29] fixing the example eon asset list --- backend/Property.py | 2 ++ backend/SearchEpc.py | 10 ++++-- etl/customers/eon/pilot_asset_list.py | 52 +++++++++++++-------------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 584e1442..6577ba62 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -111,6 +111,8 @@ class Property: self.measures = ast.literal_eval(measures) if measures else None self.uprn = epc_record.get("uprn") + self.uprn_source = self.data["uprn-source"] + self.full_sap_epc = epc_record.get("full_sap_epc") self.in_conservation_area, self.is_listed, self.is_heritage = None, None, None self.restricted_measures = False diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index e7e717ac..367d8c85 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -126,6 +126,9 @@ class SearchEpc: combinations about the home to find the property """ + # If we create the uprn based on a hash, we mark it as simulated + UPRN_SOURCE_SIMULATED = "SIMULATED" + MAX_RETRIES = 5 SUCCESS = { @@ -405,8 +408,11 @@ class SearchEpc: else: raise ValueError("Multiple UPRNs found - investigate me") - # If we do not have a UPRN, we create one based on a hash of the address & postcoce - uprn = uprns.pop() if uprns else hash(self.address1 + self.postcode) + if uprns: + uprn = uprns.pop() + else: + newest_epc["uprn-source"] = self.UPRN_SOURCE_SIMULATED + uprn = hash(self.address1 + self.postcode) if self.fast: return newest_epc, [], {}, "", "", None diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index 298b34ba..afed61e4 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -170,39 +170,35 @@ def app(): # For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free # trial with Ordnance Survey with my personal account as a temporary solution. # Let's just pull the full EPC data for this - # asset_list_with_uprn = [] - # for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): - # if row <= 104: - # continue - # time.sleep(1.1) - # searcher = SearchEpc( - # address1=property_meta["address"], - # postcode=property_meta["postcode"], - # auth_token=EPC_AUTH_TOKEN, - # os_api_key=ORDNANCE_SURVEY_API_KEY, - # full_address=", ".join([property_meta["address"], property_meta["postcode"]]) - # ) - # - # # Let's just find the UPRN - # searcher.ordnance_survey_client.get_places_api() - # - # uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] - # - # # searcher.find_property(skip_os=False) - # - # asset_list_with_uprn.append( - # { - # **property_meta, - # "uprn": uprn, - # } - # ) + asset_list_with_uprn = [] + for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): + searcher = SearchEpc( + address1=property_meta["address"], + postcode=property_meta["postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key=ORDNANCE_SURVEY_API_KEY, + full_address=", ".join([property_meta["address"], property_meta["postcode"]]) + ) + + searcher.find_property(skip_os=True) + uprn = searcher.uprn + # searcher.find_property(skip_os=False) + + asset_list_with_uprn.append( + { + **property_meta, + "uprn": uprn, + "matched_address": searcher.address1, + "matched_postcode": searcher.postcode + } + ) # Store this as a backup # import pandas as pd # asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) - # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False) + # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn_2.csv", index=False) # Read in - asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") + # asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") # Store the asset list and create the portfolio payload asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) From 37bd18642b3b1ed54f880eb55558b9697d99c3a8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 19:39:20 +0100 Subject: [PATCH 04/29] debugging eon asset list --- etl/customers/eon/pilot_asset_list.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index afed61e4..aca0884c 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -2,12 +2,11 @@ import time import pandas as pd -from utils.s3 import read_excel_from_s3 from backend.SearchEpc import SearchEpc from dotenv import load_dotenv import os from tqdm import tqdm -from utils.s3 import save_csv_to_s3 +from utils.s3 import save_csv_to_s3, read_excel_from_s3 # Read in the .env file in backend load_dotenv(dotenv_path="backend/.env") @@ -181,7 +180,10 @@ def app(): ) searcher.find_property(skip_os=True) - uprn = searcher.uprn + if searcher.newest_epc["uprn-source"] == SearchEpc.UPRN_SOURCE_SIMULATED: + uprn = None + else: + uprn = searcher.uprn # searcher.find_property(skip_os=False) asset_list_with_uprn.append( From e4aa4cbb2f1c363f56e588b5d5403fa5ca186908 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 10:48:54 +0100 Subject: [PATCH 05/29] handling edge cases for solar api --- backend/Property.py | 13 +++++- backend/apis/GoogleSolarApi.py | 58 +++++++++++++++++++++++++ backend/app/plan/router.py | 15 +++++++ etl/spatial/OpenUprnClient.py | 22 +++++++++- recommendations/recommendation_utils.py | 2 +- 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 6577ba62..eaa27359 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -18,6 +18,7 @@ from recommendations.recommendation_utils import ( get_wall_type, estimate_external_wall_area, estimate_windows, + estimate_pitched_roof_area ) from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.app.utils import sap_to_epc @@ -631,7 +632,17 @@ class Property: self.solar_panel_configuration = solar_panel_configuration # We also set the roof area - self.roof_area = roof_area + if roof_area is None: + if self.roof["is_flat"]: + self.roof_area = estimate_pitched_roof_area( + floor_area=self.insulation_floor_area, + floor_height=self.floor_height + ) + else: + self.roof_area = self.insulation_floor_area + + else: + self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): """ diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index c82c9c9a..4bb5ef37 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -589,3 +589,61 @@ class GoogleSolarApi: # we need to do is perform the solar analysis and then half the results. We set an indicator which # implies we should do this self.double_property = True + + @classmethod + def default_panel_performance(cls, property_instance): + """ + In a small number of cases, where properties have simulated uprns, we do not have a longitude and latitude + value and therefore we just return a default panel performance + :param property_instance: + :return: + """ + + cost_instance = Costs(property_instance=property_instance) + + # We return a 2.4 and 4 kwp system + panel_performance = pd.DataFrame( + [ + { + 'n_panels': 10, + 'yearly_dc_energy': 4000 * 0.99, # Assumed 99% efficient wattage -> dc + 'total_cost': cost_instance.solar_pv( + n_panels=10, has_battery=False, n_floors=property_instance.number_of_floors + )["total"], + 'weighted_ratio': None, + 'panneled_roof_area': 10 * 1.8, + 'array_wattage': 4000, + 'initial_ac_kwh_per_year': 4000 * 0.95, # Assumed 95% efficient wattage -> ac + 'lifetime_ac_kwh': None, + 'lifetime_dc_kwh': None, + 'roi': None, + 'generation_value': None, + 'generation_deficit': None, + 'expected_payback_years': None, + 'surplus': None, + 'combined_score': None, + 'rank': None + }, + { + 'n_panels': 6, + 'yearly_dc_energy': 2400 * 0.99, # Assumed 99% efficient wattage -> dc + 'total_cost': cost_instance.solar_pv( + n_panels=6, has_battery=False, n_floors=property_instance.number_of_floors + )["total"], + 'weighted_ratio': None, + 'panneled_roof_area': 6 * 1.8, + 'array_wattage': 2400, + 'initial_ac_kwh_per_year': 2400 * 0.95, # Assumed 95% efficient wattage -> ac + 'lifetime_ac_kwh': None, + 'lifetime_dc_kwh': None, + 'roi': None, + 'generation_value': None, + 'generation_deficit': None, + 'expected_payback_years': None, + 'surplus': None, + 'combined_score': None, + 'rank': None + }, + ] + ) + return panel_performance diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6e4d8475..fb053ddb 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -636,6 +636,21 @@ async def trigger_plan(body: PlanTriggerRequest): if not property_instance.is_solar_pv_valid(): continue + if unit["longitude"] is None or unit["latitude"] is None: + # At this point, we've checked that solar PV is valid, and so we provide some defaults + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": None, + "panel_performance": GoogleSolarApi.default_panel_performance( + property_instance=property_instance + ), + "unit_share_of_energy": 1 + }, + roof_area=None + ) + continue + # We check if we have a solar non-invasive recommendation if [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"]: continue diff --git a/etl/spatial/OpenUprnClient.py b/etl/spatial/OpenUprnClient.py index 11827f8d..5c43347a 100644 --- a/etl/spatial/OpenUprnClient.py +++ b/etl/spatial/OpenUprnClient.py @@ -5,6 +5,7 @@ import geopandas as gpd from utils.logger import setup_logger from utils.s3 import read_io_from_s3, save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet from backend.Property import Property +from backend.SearchEpc import SearchEpc logger = setup_logger() @@ -151,7 +152,7 @@ class OpenUprnClient: bucket_name=bucket_name, file_key="spatial/filename_meta.parquet" ) - uprns = [p.uprn for p in input_properties] + uprns = [p.uprn for p in input_properties if p.uprn_source != SearchEpc.UPRN_SOURCE_SIMULATED] uprn_map = cls.make_uprn_map(uprns, uprn_filenames) for filename, associated_uprn in tqdm(uprn_map.items(), total=len(uprn_map)): @@ -165,6 +166,9 @@ class OpenUprnClient: if p.uprn in associated_uprn: p.set_spatial(spatial_df[spatial_df["UPRN"] == p.uprn]) + if p.uprn_source == SearchEpc.UPRN_SOURCE_SIMULATED: + p.set_spatial(cls.empty_spatial_df()) + # Perform a final check to ensure that all properties have spatial data for p in input_properties: if p.spatial is None: @@ -172,6 +176,22 @@ class OpenUprnClient: return input_properties + @staticmethod + def empty_spatial_df(): + return pd.DataFrame( + [ + { + "X_COORDINATE": None, + "Y_COORDINATE": None, + "LATITUDE": None, + "LONGITUDE": None, + "conservation_status": False, + "is_listed_building": False, + "is_heritage_building": False, + } + ] + ) + @classmethod def get_spatial_data(cls, uprns: list[int], bucket_name): """ diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 883a387b..72707ded 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -655,7 +655,7 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat): return int(string_thickness) -def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float: +def estimate_pitched_roof_area(floor_area: float, floor_height: float) -> float: """ This function will estimate the area of a pitched roof, given the floor area below the roof and the floor height of the property. From 46f513a4f274db29208d5e660039cf046d6698da Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 11:29:42 +0100 Subject: [PATCH 06/29] tidying up solar api code --- backend/Property.py | 22 ++- backend/apis/GoogleSolarApi.py | 229 +++++++++++++++++++++- backend/app/plan/router.py | 177 ++--------------- recommendations/SolarPvRecommendations.py | 4 +- 4 files changed, 249 insertions(+), 183 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index eaa27359..418b0368 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -631,17 +631,23 @@ class Property: """ self.solar_panel_configuration = solar_panel_configuration + if not self.roof["is_flat"]: + default_roof_area = estimate_pitched_roof_area( + floor_area=self.insulation_floor_area, + floor_height=self.floor_height + ) + else: + default_roof_area = self.insulation_floor_area + # We also set the roof area if roof_area is None: - if self.roof["is_flat"]: - self.roof_area = estimate_pitched_roof_area( - floor_area=self.insulation_floor_area, - floor_height=self.floor_height - ) - else: - self.roof_area = self.insulation_floor_area - + self.roof_area = default_roof_area else: + # Perform a comparison between the default_roof_area and roof_area + difference = abs(default_roof_area - roof_area) + if difference / default_roof_area > 0.1: + raise Exception("Investigate difference in roof area") + self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 4bb5ef37..bf67a786 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -1,16 +1,22 @@ +import time +import requests import pandas as pd import numpy as np -from recommendations.Costs import MCS_SOLAR_PV_COST_DATA -from backend.ml_models.AnnualBillSavings import AnnualBillSavings -import requests +from typing import List from functools import lru_cache -import time -from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data -from utils.logger import setup_logger from sklearn.preprocessing import MinMaxScaler -from recommendations.Costs import Costs +from tqdm import tqdm from math import sin, cos, sqrt, atan2, radians +from utils.logger import setup_logger +from recommendations.Costs import Costs, MCS_SOLAR_PV_COST_DATA +from etl.bill_savings.EnergyConsumptionModel import EnergyConsumptionModel +from backend.ml_models.AnnualBillSavings import AnnualBillSavings +from backend.Property import Property +from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data +import backend.app.assumptions as assumptions +from backend.app.plan.schemas import PlanTriggerRequest + logger = setup_logger() @@ -590,6 +596,215 @@ class GoogleSolarApi: # implies we should do this self.double_property = True + @staticmethod + def prepare_input_data( + input_properties: List[Property], + energy_consumption_client: EnergyConsumptionModel, + body: PlanTriggerRequest + ): + """ + :param input_properties: List of properties + :param energy_consumption_client: EnergyConsumptionModel instance + :param body: PlanTriggerRequest instance + This sets up the data required to make the solar api request + :return: + """ + + building_solar_config = [ + { + "building_id": p.building_id, + "longitude": p.spatial["longitude"], + "latitude": p.spatial["latitude"], + # Energy consumption is adjusted for the property's expected post retrofit state + # We set the target rating to EPC C, which is the typical EPC rating we would expect the + # property to achieve post retrofit of just the fabric + "energy_consumption": energy_consumption_client.estimate_new_consumption( + current_energy_efficiency=p.data["current-energy-efficiency"], + target_efficiency="69", + current_consumption=p.estimate_electrical_consumption( + assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions + ) + ), + "property_id": p.id, + "uprn": p.uprn + } for p in input_properties if p.building_id is not None + ] + unit_solar_config = [ + { + "longitude": p.spatial["longitude"], + "latitude": p.spatial["latitude"], + # Energy consumption is adjusted for the property's expected post retrofit state + # We set the target rating to EPC C, which is the typical EPC rating we would expect the + # property to achieve post retrofit of just the fabric + "energy_consumption": energy_consumption_client.estimate_new_consumption( + current_energy_efficiency=p.data["current-energy-efficiency"], + target_efficiency="69", + current_consumption=p.estimate_electrical_consumption( + assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions + ), + ), + "property_id": p.id, + "uprn": p.uprn + } for p in input_properties if p.building_id is None + ] + + return building_solar_config, unit_solar_config + + @classmethod + def building_solar_analysis( + cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str + ): + """ + Perform the solar analysis for the building level + :param building_solar_config: List of building solar configurations + :param input_properties: List of properties + :param session: Database session + :param google_solar_api_key: Google Solar API key + :return: + """ + + if not building_solar_config: + return input_properties + + # Find the unique longitude and latitude pairs for each building id + unique_coordinates = {} + building_uprns = {} + for entry in building_solar_config: + building_id = entry['building_id'] + coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']} + + if building_id not in unique_coordinates: + unique_coordinates[building_id] = [] + + if coordinate_pair not in unique_coordinates[building_id]: + unique_coordinates[building_id].append(coordinate_pair) + + if building_id not in building_uprns: + building_uprns[building_id] = [] + + if entry['uprn'] not in building_uprns[building_id]: + building_uprns[building_id].append( + { + "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude'] + } + ) + + solar_panel_configuration = {} + for building_id, coordinates in unique_coordinates.items(): + if len(coordinates) > 1: + raise NotImplementedError("more than one coordinate for a building - handle me") + + coordinates = coordinates[0] + energy_consumption = sum( + [entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id] + ) + solar_api_client = cls(api_key=google_solar_api_key) + solar_api_client.get( + longitude=coordinates["longitude"], + latitude=coordinates["latitude"], + energy_consumption=energy_consumption, + is_building=True, + session=session + ) + solar_panel_configuration[building_id] = { + "insights_data": solar_api_client.insights_data, + "panel_performance": solar_api_client.panel_performance, + "n_units": len([entry for entry in building_solar_config if entry['building_id'] == building_id]) + } + + # Store the data in the database + # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it + # exists + solar_api_client.save_to_db( + session=session, uprns_to_location=building_uprns[building_id], scenario_type="building" + ) + + # Insert this into the properties that have this building id + for p in input_properties: + if p.building_id == building_id: + unit_solar_panel_configuration = solar_panel_configuration[building_id].copy() + + unit_solar_panel_configuration["unit_share_of_energy"] = ( + [x for x in building_solar_config if x["property_id"] == p.id][0]["energy_consumption"] / + energy_consumption + ) + p.set_solar_panel_configuration(unit_solar_panel_configuration) + + return input_properties + + @classmethod + def unit_solar_analysis( + cls, unit_solar_config: List, input_properties: List[Property], session, body, google_solar_api_key: str + ): + + if not unit_solar_config: + return input_properties + + # Model the solar potential at the property level + for unit in tqdm(unit_solar_config): + + # We don't need to do this if we have global inclusions that don't include solar + if body.inclusions: + if "solar_pv" not in body.inclusions: + continue + + property_instance = [p for p in input_properties if p.id == unit["property_id"]][0] + # At this level, we check if the property is suitable for solar and if now, skip + # Or if we have a solar non-invasive recommendation + if ( + (not property_instance.is_solar_pv_valid()) or + [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"] + ): + continue + + if unit["longitude"] is None or unit["latitude"] is None: + # At this point, we've checked that solar PV is valid, and so we provide some defaults + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": None, + "panel_performance": cls.default_panel_performance(property_instance=property_instance), + "unit_share_of_energy": 1 + }, + roof_area=None + ) + continue + + solar_api_client = cls(api_key=google_solar_api_key) + solar_api_client.get( + longitude=unit["longitude"], + latitude=unit["latitude"], + energy_consumption=unit["energy_consumption"], + is_building=False, + session=session, + uprn=unit["uprn"], + property_instance=property_instance + ) + + # Store the data in the database + solar_api_client.save_to_db( + session=session, + uprns_to_location=[ + { + "uprn": property_instance.uprn, + "longitude": property_instance.spatial["longitude"], + "latitude": property_instance.spatial["latitude"] + } + ], + scenario_type="unit" + ) + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": solar_api_client.insights_data, + "panel_performance": solar_api_client.panel_performance, + "unit_share_of_energy": 1 + }, + roof_area=solar_api_client.roof_area + ) + + return input_properties + @classmethod def default_panel_performance(cls, property_instance): """ diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fb053ddb..e8543930 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -513,179 +513,24 @@ async def trigger_plan(body: PlanTriggerRequest): [p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=kwh_preds) for p in input_properties] logger.info("Performing solar analysis") - # TODO: Tidy this up # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job # TODO: For simple properties, we should do a comparison/check between the solar API's roof area and the # basic estimate of roof area - building_ids = [ - { - "building_id": p.building_id, - "longitude": p.spatial["longitude"], - "latitude": p.spatial["latitude"], - # Energy consumption is adjusted for the property's expected post retrofit state - # We set the target rating to EPC C, which is the typical EPC rating we would expect the - # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( - current_energy_efficiency=p.data["current-energy-efficiency"], - target_efficiency="69", - current_consumption=p.estimate_electrical_consumption( - assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions - ) - ), - "property_id": p.id, - "uprn": p.uprn - } for p in input_properties if p.building_id is not None - ] - individual_units = [ - { - "longitude": p.spatial["longitude"], - "latitude": p.spatial["latitude"], - # Energy consumption is adjusted for the property's expected post retrofit state - # We set the target rating to EPC C, which is the typical EPC rating we would expect the - # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( - current_energy_efficiency=p.data["current-energy-efficiency"], - target_efficiency="69", - current_consumption=p.estimate_electrical_consumption( - assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions - ), - ), - "property_id": p.id, - "uprn": p.uprn - } for p in input_properties if p.building_id is None - ] - if building_ids: - # Find the unique longitude and latitude pairs for each building id - unique_coordinates = {} - building_uprns = {} - for entry in building_ids: - building_id = entry['building_id'] - coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']} + building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( + input_properties=input_properties, + energy_consumption_client=energy_consumption_client, + body=body + ) - if building_id not in unique_coordinates: - unique_coordinates[building_id] = [] - - if coordinate_pair not in unique_coordinates[building_id]: - unique_coordinates[building_id].append(coordinate_pair) - - if building_id not in building_uprns: - building_uprns[building_id] = [] - - if entry['uprn'] not in building_uprns[building_id]: - building_uprns[building_id].append( - { - "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude'] - } - ) - - solar_panel_configuration = {} - for building_id, coordinates in unique_coordinates.items(): - if len(coordinates) > 1: - raise NotImplementedError("more than one coordinate for a building - handle me") - - coordinates = coordinates[0] - energy_consumption = sum( - [entry['energy_consumption'] for entry in building_ids if entry['building_id'] == building_id] - ) - solar_api_client = GoogleSolarApi(api_key=get_settings().GOOGLE_SOLAR_API_KEY) - solar_api_client.get( - longitude=coordinates["longitude"], - latitude=coordinates["latitude"], - energy_consumption=energy_consumption, - is_building=True, - session=session - ) - solar_panel_configuration[building_id] = { - "insights_data": solar_api_client.insights_data, - "panel_performance": solar_api_client.panel_performance, - "n_units": len([entry for entry in building_ids if entry['building_id'] == building_id]) - } - - # Store the data in the database - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it - # exists - solar_api_client.save_to_db( - session=session, uprns_to_location=building_uprns[building_id], scenario_type="building" - ) - - # Insert this into the properties that have this building id - for p in input_properties: - if p.building_id == building_id: - unit_solar_panel_configuration = solar_panel_configuration[building_id].copy() - - unit_solar_panel_configuration["unit_share_of_energy"] = ( - [x for x in building_ids if x["property_id"] == p.id][0]["energy_consumption"] / - energy_consumption - ) - p.set_solar_panel_configuration(unit_solar_panel_configuration) - if individual_units: - # Model the solar potential at the property level - for unit in tqdm(individual_units): - - # TODO: Tidy up this code - # We don't need to do this if we have global inclusions that don't include solar - if body.inclusions: - if "solar_pv" not in body.inclusions: - continue - - property_instance = [p for p in input_properties if p.id == unit["property_id"]][0] - # At this level, we check if the property is suitable for solar and if now, skip - if not property_instance.is_solar_pv_valid(): - continue - - if unit["longitude"] is None or unit["latitude"] is None: - # At this point, we've checked that solar PV is valid, and so we provide some defaults - - property_instance.set_solar_panel_configuration( - solar_panel_configuration={ - "insights_data": None, - "panel_performance": GoogleSolarApi.default_panel_performance( - property_instance=property_instance - ), - "unit_share_of_energy": 1 - }, - roof_area=None - ) - continue - - # We check if we have a solar non-invasive recommendation - if [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"]: - continue - solar_api_client = GoogleSolarApi(api_key=get_settings().GOOGLE_SOLAR_API_KEY) - solar_api_client.get( - longitude=unit["longitude"], - latitude=unit["latitude"], - energy_consumption=unit["energy_consumption"], - is_building=False, - session=session, - uprn=unit["uprn"], - property_instance=property_instance - ) - - # Store the data in the database - solar_api_client.save_to_db( - session=session, - uprns_to_location=[ - { - "uprn": property_instance.uprn, - "longitude": property_instance.spatial["longitude"], - "latitude": property_instance.spatial["latitude"] - } - ], - scenario_type="unit" - ) - - property_instance.set_solar_panel_configuration( - solar_panel_configuration={ - "insights_data": solar_api_client.insights_data, - "panel_performance": solar_api_client.panel_performance, - "unit_share_of_energy": 1 - }, - roof_area=solar_api_client.roof_area - ) + input_properties = GoogleSolarApi.building_solar_analysis( + building_solar_config=building_solar_config, + input_properties=input_properties, + session=session, + google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY + ) logger.info("Identifying property recommendations") recommendations = {} diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index dc11ce4a..bb38c73c 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -2,7 +2,7 @@ import numpy as np import pandas as pd from recommendations.Costs import Costs -from recommendations.recommendation_utils import override_costs, esimtate_pitched_roof_area +from recommendations.recommendation_utils import override_costs, estimate_pitched_roof_area class SolarPvRecommendations: @@ -174,7 +174,7 @@ class SolarPvRecommendations: if self.property.roof["is_flat"]: roof_area = self.property.insulation_floor_area else: - roof_area = esimtate_pitched_roof_area( + roof_area = estimate_pitched_roof_area( floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"] ) solar_configurations = pd.DataFrame( From 33bc38fdc8bc766b855a0690dbe4d86c148772cc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 11:42:18 +0100 Subject: [PATCH 07/29] added call to unit solar api call --- backend/app/plan/router.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e8543930..8ae57c92 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -516,8 +516,6 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job - # TODO: For simple properties, we should do a comparison/check between the solar API's roof area and the - # basic estimate of roof area building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( input_properties=input_properties, @@ -532,6 +530,14 @@ async def trigger_plan(body: PlanTriggerRequest): google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY ) + input_properties = GoogleSolarApi.unit_solar_analysis( + unit_solar_config=unit_solar_config, + input_properties=input_properties, + session=session, + body=body, + google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY + ) + logger.info("Identifying property recommendations") recommendations = {} recommendations_scoring_data = [] From 97dfa3e57210065fdf954bbcdcebb8407dddcff8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 13:50:31 +0100 Subject: [PATCH 08/29] removed temp debugging code --- backend/Property.py | 11 +- backend/app/plan/router.py | 167 +------------------------ recommendations/RoofRecommendations.py | 9 +- 3 files changed, 14 insertions(+), 173 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 418b0368..55e3a912 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -639,15 +639,16 @@ class Property: else: default_roof_area = self.insulation_floor_area + # Keep a record + self.roof_area_comparison = { + "api": roof_area, + "estimated": default_roof_area + } + # We also set the roof area if roof_area is None: self.roof_area = default_roof_area else: - # Perform a comparison between the default_roof_area and roof_area - difference = abs(default_roof_area - roof_area) - if difference / default_roof_area > 0.1: - raise Exception("Investigate difference in roof area") - self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8ae57c92..9d77c1a1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -511,12 +511,12 @@ async def trigger_plan(body: PlanTriggerRequest): input_properties = OpenUprnClient.set_spatial_data(input_properties, bucket_name=get_settings().DATA_BUCKET) [p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=kwh_preds) for p in input_properties] - logger.info("Performing solar analysis") # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job + logger.info("Performing solar analysis") building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( input_properties=input_properties, energy_consumption_client=energy_consumption_client, @@ -595,171 +595,6 @@ async def trigger_plan(body: PlanTriggerRequest): scoring_epcs.extend(property_instance.updated_simulation_epcs) recommendations[property_id] = recommendations_with_impact - # For Debugging - # recommendation_impact_df = [] - # for property_id in recommendations.keys(): - # for recs_by_type in recommendations[property_id]: - # for rec in recs_by_type: - # recommendation_impact_df.append( - # { - # "property_id": property_id, - # "uprn": [p.uprn for p in input_properties if p.id == property_id][0], - # "address": [p.address for p in input_properties if p.id == property_id][0], - # "recommendation_id": rec["recommendation_id"], - # "type": rec["type"], - # "description": rec["description"], - # "sap_points": rec["sap_points"], - # "co2_equivalent_savings": rec["co2_equivalent_savings"], - # "heat_demand": rec["heat_demand"] - # } - # ) - # recommendation_impact_df = pd.DataFrame(recommendation_impact_df) - # - # surveyed_uprns = [ - # 10024087855, 121016117, 121016124, - # 10024087902, 121016121, 121016128 - # ] - # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - # # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( - # # ["windows_glazing", "internal_wall_insulation"]) - # # ] - # - # actual_impacts_df = pd.DataFrame( - # [ - # # 10024087855 - # {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, - # {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, - # {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, - # {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, - # # 121016117 - # {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, - # {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, - # {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, - # # 121016124 - # {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, - # {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, - # {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, - # # 10024087902 - # {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, - # {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, - # {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, - # # 121016121 - # {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, - # {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, - # {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, - # # 121016128 - # {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, - # {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, - # ] - # ) - # - # comparison = recommendation_impact_df.merge( - # actual_impacts_df, how="inner", on=["uprn", "type"] - # ) - # - # print(recommendation_impact_df.groupby(["uprn"])["sap_points"].sum()) - # property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] - # property = [p for p in input_properties if p.uprn == 121016128][0] - # print(property.data["current-energy-efficiency"]) - # print(property_recs["sap_points"].sum()) - # print(property_recs["type"]) - # print(float(property.data["current-energy-efficiency"]) + property_recs["sap_points"].sum()) - # recommendations[property.id][2][0]["simulation_config"] - - # from utils.s3 import read_dataframe_from_s3_parquet - # training_data = read_dataframe_from_s3_parquet( - # bucket_name="retrofit-data-dev", - # file_key="sap_change_model/2024-08-06-11-19-49/dataset_rooms.parquet" - # ) - # import pickle - # with open("delete_me.pkl", "wb") as f: - # pickle.dump(training_data, f) - - # Read in the pickle - import pickle - with open("delete_me.pkl", "rb") as f: - training_data = pickle.load(f) - - # How do we simulate windows: - ending_cols = [col for col in training_data.columns if col.endswith("_ending")] - starting = {} - for c in ending_cols: - starting_colname = c.replace("_ending", "_starting") - if starting_colname in training_data.columns: - starting[c] = starting_colname - else: - starting[c] = c.replace("_ending", "") - - allowed_to_change = [ - # Windows - "windows_energy_eff_ending", - "glazed_type_ending", - "glazing_type_ending", - "multi_glaze_proportion_ending", - - # Other - "sap_ending", - "heat_demand_ending", - "carbon_ending", - "estimated_perimeter_ending", - "lodgement_year_ending", - "lodgement_month_ending", - "days_to_ending", - "number_habitable_rooms_ending", - "number_heated_rooms_ending", - ] - fixed = [c for c in ending_cols if c not in allowed_to_change + ["uprn"]] - training_fixed = training_data.copy() - for col in fixed: - starting_col = starting[col] - training_fixed = training_fixed[training_fixed[col] == training_fixed[starting_col]] - - training_fixed = training_fixed.reset_index(drop=True) - - # Get the recommendation config for this uprn - uprn = 121016121 - property_instance = [p for p in input_properties if p.uprn == uprn][0] - property_recs = recommendations[property_instance.id] - window_recs = [r for r in property_recs if r[0]["type"] == "windows_glazing"][0] - window_recs[0].keys() - window_recs[0]["description_simulation"]["multi-glaze-proportion"] - # TODO: - In description_simulation for windows, we update glazed-type but in the model training data there - # is a column called "glazing-type". - # - We don't update glazed-area (should be "Much More Than Typical" most likely? Or Normal??) - # TODO: I think we update eveything that we actually need to, when simulating the recommendation impact for the - # ML models - # TODO: Secondary glazing appears to go to "Good", not "Average". Investigate why - # TODO: For the two properties, force recommendations for double glazing and check impact - - z = training_data[training_data["glazed_type_ending"] == "secondary glazing"] - z = z[z["multi_glaze_proportion_ending"] == 100] - z["windows_energy_eff_ending"].value_counts() - - # Find the things that change - example = training_fixed.iloc[3] - for _, example in training_fixed.iterrows(): - things_that_change = [] - for c in ending_cols: - if example[c] != example[starting[c]]: - things_that_change.append(c) - if len(things_that_change) > 4: - print(things_that_change) - print(example["uprn"]) - # blah - - # 100051011370 (doesn't change in actual glazing) - # example["glazed_type_ending"] - # double glazing installed before 2002 - # example["glazed_type_starting"] - # double glazing, unknown install date - - # 100040925015 - # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) scoring_epcs = kwh_client.transform(data=scoring_epcs, cleaned=cleaned) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 2a77a3a5..4d22eb2d 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -84,12 +84,17 @@ class RoofRecommendations: return (self.insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"] - def is_room_roof_insulated(self): + def is_room_roof_insulated_or_unsuitable(self, measures): """ Check if the room roof is already insulated """ + # If the roof is a room roof room roof is not included in the measures, we deem the recommendation unsuitable + unsuitable = "room_roof_insulation" not in measures and self.property.roof["is_roof_room"] + if unsuitable: + return True + full_insulated_room_roof = ( self.property.roof["is_roof_room"] and self.property.roof["insulation_thickness"] in ["average", "above_average"] @@ -123,7 +128,7 @@ class RoofRecommendations: if (self.insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]: return - if self.is_room_roof_insulated(): + if self.is_room_roof_insulated_or_unsuitable(measures): return # If we have a u-value already, need to implement this From 9beb8b7a24bcc273a61e11c3d40d85bf9de19062 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:05:04 +0100 Subject: [PATCH 09/29] new fuel type to efficiency mapped --- backend/app/assumptions.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index 5f8cb85c..80baa69f 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -1,7 +1,7 @@ # Assumes that the average efficiency of an air source heat pump is 250%, taking the median of the 200-400% range, # which is often quoted as a sensible efficiency range for air source heat pumps. PESSIMISTIC_ASHP_EFFICIENCY = 200 -AVERAGE_ASHP_EFFICIENCY = 300 +AVERAGE_ASHP_EFFICIENCY = 250 # Conservative estimate of the proportion of electricity that will be consumed, whereas the rest will # be exported @@ -11,34 +11,36 @@ DESCRIPTIONS_TO_FUEL_TYPES = { "Air source heat pump, radiators, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 }, - "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, + "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, 'Electric storage heaters': {"fuel": 'Electricity', "cop": 1}, "Electric immersion, off-peak": {"fuel": 'Electricity', "cop": 1}, "Electric storage heaters, radiators": {"fuel": 'Electricity', "cop": 1}, "Room heaters, electric": {"fuel": 'Electricity', "cop": 1}, "Electric immersion, standard tariff": {"fuel": 'Electricity', "cop": 1}, "Portable electric heaters assumed for most rooms": {"fuel": 'Electricity', "cop": 1}, - "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.9}, + "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.85}, "Room heaters, dual fuel (mineral and wood)": {"fuel": 'Wood Logs', "cop": 1}, - "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.9}, + "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.85}, "Warm air, Electricaire": {"fuel": "Electricity", "cop": 1}, - "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.9}, - "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.9}, + "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.85}, + "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.85}, "No system present: electric heaters assumed": {"fuel": "Electricity", "cop": 1}, "Electric instantaneous at point of use": {"fuel": "Electricity", "cop": 1}, - "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.9}, + "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.85}, "Electric storage heaters, Electric storage heaters": {"fuel": "Electricity", "cop": 1}, - "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.9}, - "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.9}, - "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.9}, + "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.85}, + "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.85}, + "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.85}, "Electric immersion, standard tariff, plus solar": {"fuel": "Electricity + Solar Thermal", "cop": 1}, - "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.9}, + "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.85}, "Electric underfloor heating": {"fuel": "Electricity", "cop": 1}, "No system present: electric immersion assumed": {"fuel": "Electricity", "cop": 1}, "Air source heat pump, underfloor, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 }, + "Gas instantaneous at point of use": {"fuel": "Natural Gas", "cop": 0.85}, + "Room heaters, wood logs": {"fuel": "Wood Logs", "cop": 1}, } From aadcc56d32547addb06af616a192b5c1446b03fd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:33:29 +0100 Subject: [PATCH 10/29] adding database queries to Outputs class --- backend/Outputs.py | 102 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 backend/Outputs.py diff --git a/backend/Outputs.py b/backend/Outputs.py new file mode 100644 index 00000000..a846965b --- /dev/null +++ b/backend/Outputs.py @@ -0,0 +1,102 @@ +from sqlalchemy.orm import sessionmaker + +from backend.app.db.connection import db_engine +from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel +from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario + + +class Outputs: + FORMATS = ["mds"] + + def __init__(self, format, portfolio_id): + """ + This class handles the creation of standard outputs for the backend. For example, creation of + an excel output, to be used for the MDS data sheet, required by E.ON + + :param format: The format of the output, e.g. mds + :param portfolio_id: The id of the portfolio for which the output is being created + """ + + if format not in self.FORMATS: + raise ValueError("Invalid format, should be one of {}".format(self.FORMATS)) + + self.format = format + self.portfolio_id = portfolio_id + + # Connect to the database + self.session = sessionmaker(bind=db_engine)() + + def get_properties_from_db(self): + # Get properties and their details for a specific portfolio + self.session.begin() + properties_query = self.session.query( + PropertyModel, + PropertyDetailsEpcModel + ).join( + PropertyDetailsEpcModel, + PropertyModel.id == PropertyDetailsEpcModel.property_id + ).filter( + PropertyModel.portfolio_id == self.portfolio_id # Filter by portfolio ID + ).all() + + # Transform properties data to include all fields dynamically + properties_data = [ + {**{col.name: getattr(prop.PropertyModel, col.name) for col in PropertyModel.__table__.columns}, + **{col.name: getattr(prop.PropertyDetailsEpcModel, col.name) for col in + PropertyDetailsEpcModel.__table__.columns}} + for prop in properties_query + ] + + self.session.close() + return properties_data + + def get_plans_from_db(self): + + self.session.begin() + + plans_query = self.session.query(Plan).all() + # Transform plans data to include all fields dynamically + plans_data = [ + {col.name: getattr(plan, col.name) for col in Plan.__table__.columns} + for plan in plans_query + ] + + self.session.close() + return plans_data + + def export_mds(self): + """ + This function will export the data in the MDS format + Core data required: + - Property address + - Property postcode + - uprn + - recommended measures + - pre-EPC + - pre-SAP + - pre Heat Demand + - Property Type + - Built form + - Wall type + - Tenure + - Fuel type + - Estimated bill + - Recommended measures + - Post EPC + - Post heat demand + - Bill savings + - Kwh savings + """ + + properties_data = self.get_properties_from_db() + + plans_data = self.get_plans_from_db() + + plan_ids = [plan['id'] for plan in plans_data] + + def export(self): + """ + This function will export the data in the required format + """ + if self.format == "mds": + self.export_mds() From 6bc2ee0a6cdfc2586b7c64672debe8067f69ab4e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:39:04 +0100 Subject: [PATCH 11/29] adding rest of database query functions --- backend/Outputs.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index a846965b..0be2ad3b 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel -from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario +from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations class Outputs: @@ -28,7 +28,6 @@ class Outputs: def get_properties_from_db(self): # Get properties and their details for a specific portfolio - self.session.begin() properties_query = self.session.query( PropertyModel, PropertyDetailsEpcModel @@ -47,13 +46,10 @@ class Outputs: for prop in properties_query ] - self.session.close() return properties_data def get_plans_from_db(self): - self.session.begin() - plans_query = self.session.query(Plan).all() # Transform plans data to include all fields dynamically plans_data = [ @@ -61,9 +57,36 @@ class Outputs: for plan in plans_query ] - self.session.close() return plans_data + def get_recommendations_from_db(self, plan_ids): + # Get recommendations through PlanRecommendations for those plans and that are default + recommendations_query = self.session.query( + Recommendation, + Plan.scenario_id + ).join( + PlanRecommendations, Recommendation.id == PlanRecommendations.recommendation_id + ).join( + Plan, Plan.id == PlanRecommendations.plan_id # Join with Plan to access scenario_id + ).filter( + PlanRecommendations.plan_id.in_(plan_ids), + Recommendation.default == True # Filtering for default recommendations + ).all() + + # Transform recommendations data to include all fields dynamically and include scenario_id + recommendations_data = [ + { + **{ + col.name: getattr(rec.Recommendation, col.name) if + hasattr(rec, 'Recommendation') else getattr(rec, col.name) + for col in Recommendation.__table__.columns + }, + "Scenario ID": rec.scenario_id + } for rec in recommendations_query + ] + + return recommendations_data + def export_mds(self): """ This function will export the data in the MDS format @@ -88,12 +111,16 @@ class Outputs: - Kwh savings """ + self.session.begin() properties_data = self.get_properties_from_db() plans_data = self.get_plans_from_db() plan_ids = [plan['id'] for plan in plans_data] + recommendations_data = self.get_recommendations_from_db(plan_ids) + self.session.close() + def export(self): """ This function will export the data in the required format From ad82b15c6efc59d60f3ab184392d16abcd9b3cbf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 17:29:04 +0100 Subject: [PATCH 12/29] added measure matrix code --- backend/Outputs.py | 153 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index 0be2ad3b..4e300e81 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -1,5 +1,7 @@ +import pandas as pd from sqlalchemy.orm import sessionmaker +from backend.app.utils import sap_to_epc from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations @@ -8,6 +10,37 @@ from backend.app.db.models.recommendations import Recommendation, Plan, PlanReco class Outputs: FORMATS = ["mds"] + MDS_MEASURE_MAPPING = { + "external_wall_insulation": "EWI (Trad Const)", + "cavity_wall_insulation": "CWI", + "loft_insulation": "LI", + "party_wall_insulation": "Party Wall Insu", + "internal_wall_insulation": "IWI (POA - Prov Sum Only)", + "suspended_floor_insulation": "U/F Insu (Manual install)", + "solid_floor_insulation": "Solid floor insl (Out of scope - Prov sum only)", + "air_source_heat_pump": "ASHP Htg", + "ground_source_heat_pump": "GSHP Htg", + "shared_ground_loops": "Shared ground loops", + "communal_heat_networks": "Communal heat networks", + "district_heating_networks": "District heating networks", + "high_heat_retention_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)", + "low_energy_lighting": "Low Energy Bulbs", + "cylinder_insulation": "Cyl Insulation", + "smart_controls": "Smart controls", + "zone_controls": "Zone controls", + "trvs": "Upgrade TRV's", + "solar_pv": "Solar PV", + "solar_thermal": "Solar Thermal", + "double_glazing": "Double Glazing (POA - Prov sum only)", + "draught_proofing": "Draught Proofing", + "mechanical_ventilation": "Ventilation upgrade", + "gas_boiler": "Gas Boiler Replacement", + "flat_roof_insulation": "Flat roof (Out of scope - prov sum only)", + "room_in_roof_insulation": "RIR (POA - Prov sum only)", + "ev_charging": "EV Charging", + "battery": "Battery" + } + def __init__(self, format, portfolio_id): """ This class handles the creation of standard outputs for the backend. For example, creation of @@ -50,7 +83,7 @@ class Outputs: def get_plans_from_db(self): - plans_query = self.session.query(Plan).all() + plans_query = self.session.query(Plan).filter(Plan.portfolio_id == self.portfolio_id).all() # Transform plans data to include all fields dynamically plans_data = [ {col.name: getattr(plan, col.name) for col in Plan.__table__.columns} @@ -87,6 +120,41 @@ class Outputs: return recommendations_data + def make_mds_measure_matrix(self, scenario_recommendations): + all_measures = list(self.MDS_MEASURE_MAPPING.values()) + + # Collect rows in a list + rows = [] + + # Populate the rows list + for idx, row in scenario_recommendations.iterrows(): + property_id = row["property_id"] + measure_type = row["measure_type"] + + # Get the label for the current type + measure_label = self.MDS_MEASURE_MAPPING.get(measure_type, None) + + # If the property_id already exists in the collected rows, update it + existing_row = next((item for item in rows if item["property_id"] == property_id), None) + if existing_row is None: + # Create a new row if the property_id doesn't exist + new_row = {measure: None for measure in all_measures} + new_row["property_id"] = property_id + rows.append(new_row) + else: + new_row = existing_row + + # Set the corresponding measure label in the row + new_row[measure_label] = measure_label + + # Convert the list of dictionaries to a DataFrame + matrix = pd.DataFrame(rows) + + # Reset the index for cleanliness + matrix.reset_index(drop=True, inplace=True) + + return matrix + def export_mds(self): """ This function will export the data in the MDS format @@ -115,12 +183,93 @@ class Outputs: properties_data = self.get_properties_from_db() plans_data = self.get_plans_from_db() - plan_ids = [plan['id'] for plan in plans_data] recommendations_data = self.get_recommendations_from_db(plan_ids) self.session.close() + # Convert these tables to dataframes + properties_df = pd.DataFrame(properties_data) + plans_df = pd.DataFrame(plans_data) + recommendations_df = pd.DataFrame(recommendations_data) + + scenario_ids = plans_df["scenario_id"].unique() + + # We start to create the MDS sheet + mds = properties_df[ + [ + "property_id", + "address", + "postcode", + "uprn", + "current_epc_rating", + "current_sap_points", + # TODO: Need to add current heat demand + "property_type", + "built_form", + "total_floor_area", + "walls", + "tenure", + "mainfuel", + # TODO: For estimated bill, this should probably be without the cost of appliances + ] + ].copy().rename( + columns={ + "address": "Address", + "postcode": "Postcode", + "uprn": "UPRN", + "current_epc_rating": "Pre EPC", + "current_sap_points": "EPC Source", + # TODO: Need to add current heat demand + "property_type": "Property Type", + "built_form": "Built Form", + "total_floor_area": "Floor area m2 (If known)", + "walls": "Wall Type (Mandatory field)", + "tenure": "Tenure", + "mainfuel": "Existing Fuel Type" + # TODO: For estimated bill, this should probably be without the cost of appliances + } + ) + + # TODO - format + # 1) property type + # 2) walls + # 3) tenure + # 4) mainfuel + # 5) Epc Rating + + mds_output_by_scenario = {} + for scenario_id in scenario_ids: + scenario_recommendations = recommendations_df[recommendations_df["Scenario ID"] == scenario_id] + + # For each measure, we create the measure matrix + scenario_measure_matrix = self.make_mds_measure_matrix(scenario_recommendations) + + # Calculate the predicted impact on: SAP, heat demand, bills, kwh + recommendation_impacts = scenario_recommendations.groupby("property_id")[ + ["sap_points", "heat_demand", "kwh_savings", "energy_cost_savings"] + ].sum().reset_index() + + scenario_mds = mds.merge( + recommendation_impacts, how="left", on="property_id" + ) + # If we have no recommendations, sap_points, kwh_savings, head_demand will be NaN + scenario_mds.fillna(0, inplace=True) + scenario_mds["Post SAP"] = scenario_mds["EPC Source"] + scenario_mds["sap_points"] + # Round Post SAP down to the nearest integer + scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x)) + scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x)) + + # TODO: Post heat demand + + scenario_mds = scenario_mds.rename( + columns={ + "sap_points": "Predicted SAP Points", + "kwh_savings": "Energy Saving (Kwh)", + "energy_cost_savings": "Bill Reduction (£ per yr)" + } + ) + def export(self): """ This function will export the data in the required format From 17f653340bb26a098377c8415496284ccf47d360 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 17:31:58 +0100 Subject: [PATCH 13/29] almost completed the mds output --- backend/Outputs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/Outputs.py b/backend/Outputs.py index 4e300e81..aef47b54 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -251,9 +251,14 @@ class Outputs: ].sum().reset_index() scenario_mds = mds.merge( + scenario_measure_matrix, how="left", on="property_id" + ).merge( recommendation_impacts, how="left", on="property_id" ) # If we have no recommendations, sap_points, kwh_savings, head_demand will be NaN + to_clean = [c for c in recommendation_impacts.columns if c != "property_id"] + for col in to_clean: + scenario_mds[col].fillna(0, inplace=True) scenario_mds.fillna(0, inplace=True) scenario_mds["Post SAP"] = scenario_mds["EPC Source"] + scenario_mds["sap_points"] # Round Post SAP down to the nearest integer From 2a75a6db17c625ac9f4a78a55297a781c855beab Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 20:19:25 +0100 Subject: [PATCH 14/29] adding heat demand to mds outputs --- backend/Outputs.py | 2 +- backend/app/db/functions/recommendations_functions.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index aef47b54..9cc3e38c 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -220,7 +220,7 @@ class Outputs: "uprn": "UPRN", "current_epc_rating": "Pre EPC", "current_sap_points": "EPC Source", - # TODO: Need to add current heat demand + "primary_energy_consumption": "Existing Heating Demand Kwh/m2/y", "property_type": "Property Type", "built_form": "Built Form", "total_floor_area": "Floor area m2 (If known)", diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index b03909ee..5f6791f2 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -121,6 +121,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], "already_installed": rec["already_installed"], + "head_demand": rec["heat_demand"] } for rec in recommendations_to_upload ] From 1f1bf9981ce2e0b3a9bf42a8d096fb9aba333f4e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:38:14 +0100 Subject: [PATCH 15/29] adding measure type for wall and roof insulation --- backend/Property.py | 8 ++++++-- backend/app/db/models/portfolio.py | 7 +++++++ backend/app/plan/router.py | 11 ++++++----- recommendations/Recommendations.py | 17 +++++++++-------- recommendations/RoofRecommendations.py | 8 +++++--- recommendations/WallRecommendations.py | 2 ++ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 55e3a912..491a886f 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -621,7 +621,7 @@ class Property: self.set_windows_count() self.set_energy_source() self.find_energy_sources() - self.set_current_energy_bill(kwh_client, kwh_predictions) + self.set_current_energy(kwh_client, kwh_predictions) def set_solar_panel_configuration( self, solar_panel_configuration, roof_area @@ -651,7 +651,7 @@ class Property: else: self.roof_area = roof_area - def set_current_energy_bill(self, kwh_client, kwh_predictions): + def set_current_energy(self, kwh_client, kwh_predictions): """ Given what we know about the property now, estimates the current energy consumption using the UCL paper https://www.sciencedirect.com/science/article/pii/S0378778823002542 @@ -808,6 +808,9 @@ class Property: def get_property_details_epc(self, portfolio_id: int, rating_lookup): + if self.current_energy_bill is None: + raise ValueError("Current energy bill has not been set") + property_details_epc = { "property_id": self.id, "portfolio_id": portfolio_id, @@ -865,6 +868,7 @@ class Property: "current_energy_demand": self.current_energy_consumption, "current_energy_demand_heating_hotwater": self.current_energy_consumption_heating_hotwater, "estimated": self.data.get("estimated", False), + **self.current_energy_bill } return property_details_epc diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index e2b258f4..5f51cf46 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -173,6 +173,13 @@ class PropertyDetailsEpcModel(Base): current_energy_demand = Column(Float) current_energy_demand_heating_hotwater = Column(Float) estimated = Column(Boolean, default=False) + # Include estimates for energy bills, across the different types of energy + heating_cost_current = Column(Float) + hot_water_cost_current = Column(Float) + lighting_cost_current = Column(Float) + appliances_cost_current = Column(Float) + gas_standing_charge = Column(Float) + electricity_standing_charge = Column(Float) class PropertyDetailsSpatial(Base): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 9d77c1a1..6b8c5bea 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -10,7 +10,6 @@ from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import sessionmaker from starlette.responses import Response -import backend.app.assumptions as assumptions from backend.app.config import get_settings, get_prediction_buckets from backend.app.db.connection import db_engine from backend.app.db.functions.materials_functions import get_materials @@ -614,10 +613,12 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations = recommendations.get(property_id, []) property_instance = [p for p in input_properties if p.id == property_id][0] - property_current_energy_bill = Recommendations.calculate_recommendation_tenant_savings( - property_instance=property_instance, - kwh_simulation_predictions=kwh_simulation_predictions, - property_recommendations=property_recommendations + property_current_energy_bill = ( + Recommendations.calculate_recommendation_tenant_savings( + property_instance=property_instance, + kwh_simulation_predictions=kwh_simulation_predictions, + property_recommendations=property_recommendations + ) ) property_instance.current_energy_bill = property_current_energy_bill diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 526cb2a2..78725c2d 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -797,13 +797,14 @@ class Recommendations: electricity_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_ELECTRICITY * 365 - current_energy_bill = ( - starting_figures["heating_cost"] + - starting_figures["hotwater_cost"] + - property_instance.energy_cost_estimates["unadjusted"]["lighting"] + - property_instance.energy_cost_estimates["unadjusted"]["appliances"] + - gas_standing_charge + - electricity_standing_charge - ) + # We return a dictionary that contains the individual costs, that can be stored to the database + current_energy_bill = { + "heating_cost_current": starting_figures["heating_cost"], + "hot_water_cost_current": starting_figures["hotwater_cost"], + "lighting_cost_current": property_instance.energy_cost_estimates["unadjusted"]["lighting"], + "appliances_cost_current": property_instance.energy_cost_estimates["unadjusted"]["appliances"], + "gas_standing_charge": gas_standing_charge, + "electricity_standing_charge": electricity_standing_charge, + } return current_energy_bill diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 4d22eb2d..3e266cee 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -250,8 +250,10 @@ class RoofRecommendations: if is_pitched: insulation_materials = self.loft_insulation_materials + measure_type = "loft_insulation" elif is_flat: insulation_materials = self.flat_roof_insulation_materials + measure_type = "flat_roof_insulation" else: raise ValueError("Roof is not pitched or flat") @@ -367,6 +369,7 @@ class RoofRecommendations: ) ], "type": material["type"], + "measure_type": measure_type, "description": self.make_roof_insulation_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, @@ -490,10 +493,9 @@ class RoofRecommendations: recommendations.append( { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "room_roof_insulation", + "measure_type": "room_roof_insulation", "description": "Insulate room in roof at rafters and re-decorate", "starting_u_value": u_value, "new_u_value": new_u_value, diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 4902ae03..dd5e861c 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -374,6 +374,7 @@ class WallRecommendations(Definitions): ) ], "type": "cavity_wall_insulation", + "measure_type": "cavity_wall_insulation", "description": description, "starting_u_value": u_value, "new_u_value": new_u_value, @@ -545,6 +546,7 @@ class WallRecommendations(Definitions): ) ], "type": material["type"], + "measure_type": material["type"], # This is distinguished between EWI & IWI "description": self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, From 0c2463efe00ddd266f5f239dd743f1b6c2a46608 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:39:59 +0100 Subject: [PATCH 16/29] added measure type for ventilation, draught proofing --- recommendations/DraughtProofingRecommendations.py | 1 + recommendations/VentilationRecommendations.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/recommendations/DraughtProofingRecommendations.py b/recommendations/DraughtProofingRecommendations.py index 197d80cc..4bd85a03 100644 --- a/recommendations/DraughtProofingRecommendations.py +++ b/recommendations/DraughtProofingRecommendations.py @@ -38,6 +38,7 @@ class DraughtProofingRecommendations: "phase": None, "parts": [], "type": "draught_proofing", + "measure_type": "draught_proofing", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 5913ab9c..9738b898 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -66,6 +66,7 @@ class VentilationRecommendations(Definitions): "phase": None, "parts": part, "type": part[0]["type"], + "measure_type": "mechanical_ventilation", "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, @@ -106,6 +107,7 @@ class VentilationRecommendations(Definitions): "phase": None, "parts": [], "type": "trickle_vents", + "measure_type": "trickle_vents", "description": description, "starting_u_value": None, "new_u_value": None, From b3b2021a1bb96a711208a673286e1211761f0c82 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:41:42 +0100 Subject: [PATCH 17/29] added floors and leds measure types --- recommendations/FloorRecommendations.py | 1 + recommendations/LightingRecommendations.py | 1 + 2 files changed, 2 insertions(+) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index d82162da..25741e7a 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -241,6 +241,7 @@ class FloorRecommendations(Definitions): ), ], "type": material["type"], + "measure_type": material["type"], # This is distinct between suspended and solid floor "description": self._make_floor_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 2b0e8724..f9a1d63a 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -152,6 +152,7 @@ class LightingRecommendations: "phase": phase, "parts": [], "type": "low_energy_lighting", + "measure_type": "low_energy_lighting", "description": description, "starting_u_value": None, "new_u_value": None, From e2d0f920b0dfa358d4019dca4fcffecd5dab87fa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:42:24 +0100 Subject: [PATCH 18/29] added windows measure_type --- recommendations/WindowsRecommendations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index cd1b982b..9b85b316 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -223,6 +223,7 @@ class WindowsRecommendations: "phase": phase, "parts": [], "type": "windows_glazing", + "measure_type": "double_glazing" if not is_secondary_glazing else "secondary_glazing", "description": description, "starting_u_value": None, "new_u_value": None, From 2b992fc9f5ddb3d394090d7ffebdc9840085d56d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:43:06 +0100 Subject: [PATCH 19/29] added measure type for minor measures --- recommendations/FireplaceRecommendations.py | 1 + recommendations/WindowsRecommendations.py | 1 + 2 files changed, 2 insertions(+) diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 163728dd..60802bb6 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -41,6 +41,7 @@ class FireplaceRecommendations(Definitions): "phase": phase, "parts": [], "type": "sealing_open_fireplace", + "measure_type": "sealing_open_fireplace", "description": "Seal %s open fireplaces" % str(number_open_fireplaces), "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 9b85b316..343965e3 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -273,6 +273,7 @@ class WindowsRecommendations: "phase": phase, "parts": [], "type": "mixed_glazing", + "measure_type": "mixed_glazing", "description": description, "starting_u_value": None, "new_u_value": None, From 9b6f8c486458b0dfff3b838bac5e5aca713c45cf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:51:22 +0100 Subject: [PATCH 20/29] added measure type to heating recommendations --- recommendations/HeatingRecommender.py | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index b54f89bb..b4ca9a49 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -220,6 +220,8 @@ class HeatingRecommender: for k in ["total", "subtotal", "vat", "labour_hours", "labour_days"]: combined_rec[k] = rec[k] + rec2[k] + combined_rec["measure_type"] = "+".join([rec["measure_type"], rec2["measure_type"]]) + combined_recommendations.append(combined_rec) self.heating_recommendations.extend(combined_recommendations) @@ -512,10 +514,9 @@ class HeatingRecommender: ashp_recommendation = { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": "air_source_heat_pump", "description": description, "starting_u_value": None, "new_u_value": None, @@ -556,7 +557,8 @@ class HeatingRecommender: phase, heating_controls_only, system_change, - system_type + system_type, + measure_type ): """ Given a recommendation for heating controls, and a recommendation for the heating system, we combine the two @@ -572,7 +574,8 @@ class HeatingRecommender: current system. If we have a system change and we have a heat control recommendation, we only recommend both heating and controls together :param system_type: The type of heating system we are recommending - :return: + :param measure_type: The type of measure we are recommending - more granular than the "type" field, allowing us + to distinguish between different types of heating recommendations """ # We produce recommendations with & without heating controls @@ -616,10 +619,9 @@ class HeatingRecommender: recommendation = { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": measure_type, "description": recommendation_description, "starting_u_value": None, "new_u_value": None, @@ -799,7 +801,8 @@ class HeatingRecommender: phase=phase, heating_controls_only=heating_controls_only, system_change=system_change, - system_type="high_heat_retention_storage_heater" + system_type="high_heat_retention_storage_heater", + measure_type="high_heat_retention_storage_heater" ) if _return: return recommendations @@ -977,10 +980,9 @@ class HeatingRecommender: boiler_recommendation = { "phase": recommendation_phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": "boiler_upgrade", "description": description, "starting_u_value": None, "new_u_value": None, @@ -1027,7 +1029,8 @@ class HeatingRecommender: phase=recommendation_phase, heating_controls_only=False, system_change=True, - system_type="boiler_upgrade" + system_type="boiler_upgrade", + measure_type="boiler_upgrade", ) combined_recommendations.extend(combined_recommendation) From feb60537c0ebcfb567208d369c2ceb46be4cbece Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:54:46 +0100 Subject: [PATCH 21/29] added measure type to some controls recommendations --- backend/app/plan/schemas.py | 1 + recommendations/HeatingControlRecommender.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 6f0f6327..447e0da2 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -64,6 +64,7 @@ MEASURE_MAP = { "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"], "heating": ["boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"], "windows": ["double_glazing", "secondary_glazing"], + "heating_controls": ["roomstat_programmer_trvs", "time_temperature_zone_control"] } diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 62e292df..c613aa42 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -216,6 +216,7 @@ class HeatingControlRecommender: self.recommendation.append( { "type": "heating_control", + "measure_type": "roomstat_programmer_trvs", "parts": [], "description": description, **cost_result, @@ -289,6 +290,7 @@ class HeatingControlRecommender: self.recommendation.append( { "type": "heating_control", + "measure_type": "time_temperature_zone_control", "parts": [], "description": description, **cost_result, From 79746e9a3643f9e18b78dce8081208238ba758a3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:59:40 +0100 Subject: [PATCH 22/29] finished adding measure_type to recommendations and added check to ensure it's populated --- recommendations/HotwaterRecommendations.py | 6 +++--- recommendations/Recommendations.py | 5 +++++ recommendations/SecondaryHeating.py | 1 + recommendations/SolarPvRecommendations.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/recommendations/HotwaterRecommendations.py b/recommendations/HotwaterRecommendations.py index 0d34c894..636a7be0 100644 --- a/recommendations/HotwaterRecommendations.py +++ b/recommendations/HotwaterRecommendations.py @@ -58,10 +58,9 @@ class HotwaterRecommendations: self.recommendations.append( { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "hot_water_tank_insulation", + "measure_type": "hot_water_tank_insulation", "description": description, "starting_u_value": None, "new_u_value": None, @@ -107,6 +106,7 @@ class HotwaterRecommendations: "phase": phase, "parts": [], "type": "cylinder_thermostat", + "measure_type": "cylinder_thermostat", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 78725c2d..1b152238 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -267,6 +267,11 @@ class Recommendations: property_recommendations, ) + # Check to make sure measure_type is populated + for recs in property_recommendations: + if any(pd.isnull(rec.get("measure_type")) for rec in recs): + raise ValueError("Measure type is not populated") + return property_recommendations, property_representative_recommendations @staticmethod diff --git a/recommendations/SecondaryHeating.py b/recommendations/SecondaryHeating.py index aed48da2..7c20bcdd 100644 --- a/recommendations/SecondaryHeating.py +++ b/recommendations/SecondaryHeating.py @@ -52,6 +52,7 @@ class SecondaryHeating: "phase": phase, "parts": [], "type": "secondary_heating", + "measure_type": "secondary_heating", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index bb38c73c..08f077d2 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -126,6 +126,7 @@ class SolarPvRecommendations: "phase": phase, "parts": [], "type": "solar_pv", + "measure_type": "solar_pv", "description": description, "starting_u_value": None, "new_u_value": None, @@ -221,6 +222,7 @@ class SolarPvRecommendations: "phase": phase, "parts": [], "type": "solar_pv", + "measure_type": "solar_pv", "description": description, "starting_u_value": None, "new_u_value": None, From 4ef2f0af28a31dae746b5c99485688826ca0db02 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 12:10:45 +0100 Subject: [PATCH 23/29] completed mds outputs for the moment --- backend/Outputs.py | 84 ++++++++++++++++--- .../db/functions/recommendations_functions.py | 3 +- backend/app/db/models/recommendations.py | 1 + backend/app/plan/router.py | 4 +- 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index 9cc3e38c..f9538709 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -1,6 +1,10 @@ +import msgpack import pandas as pd +import numpy as np from sqlalchemy.orm import sessionmaker +from datetime import datetime +from utils.s3 import read_from_s3, save_excel_to_s3 from backend.app.utils import sap_to_epc from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel @@ -55,10 +59,19 @@ class Outputs: self.format = format self.portfolio_id = portfolio_id + self.today = datetime.now().strftime("%Y-%m-%d") # Connect to the database self.session = sessionmaker(bind=db_engine)() + # Download cleaned data + self.cleaned_epc_lookup = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + self.cleaned_epc_lookup = msgpack.unpackb(self.cleaned_epc_lookup, raw=False) + def get_properties_from_db(self): # Get properties and their details for a specific portfolio properties_query = self.session.query( @@ -204,14 +217,19 @@ class Outputs: "uprn", "current_epc_rating", "current_sap_points", - # TODO: Need to add current heat demand + "primary_energy_consumption", "property_type", "built_form", "total_floor_area", "walls", "tenure", "mainfuel", - # TODO: For estimated bill, this should probably be without the cost of appliances + # The bills columns are split out - we include them and aggregate, without appliances + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" ] ].copy().rename( columns={ @@ -226,17 +244,46 @@ class Outputs: "total_floor_area": "Floor area m2 (If known)", "walls": "Wall Type (Mandatory field)", "tenure": "Tenure", - "mainfuel": "Existing Fuel Type" - # TODO: For estimated bill, this should probably be without the cost of appliances } ) - # TODO - format - # 1) property type - # 2) walls - # 3) tenure - # 4) mainfuel - # 5) Epc Rating + mds["Estimated bill (£ per year)"] = ( + mds["heating_cost_current"] + + mds["hot_water_cost_current"] + + mds["lighting_cost_current"] + + mds["gas_standing_charge"] + + mds["electricity_standing_charge"] + ) + + mds = mds.drop( + columns=[ + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" + ] + ) + + # Formatting - Pre EPC is an enum + mds["Pre EPC"] = [x.value for x in mds["Pre EPC"].values] + mds["Wall Type (Mandatory field)"] = mds["Wall Type (Mandatory field)"].str.split(",").str[0] + # Remove average thermal transmittance field + mds["Wall Type (Mandatory field)"] = np.where( + mds["Wall Type (Mandatory field)"].str.contains("Average thermal transmittance"), + "", + mds["Wall Type (Mandatory field)"] + ) + + mds = mds.merge( + pd.DataFrame(self.cleaned_epc_lookup["main-fuel"])[["clean_description", "fuel_type"]], + left_on="mainfuel", + right_on="clean_description", + how="left" + ) + mds = mds.rename(columns={"fuel_type": "Existing Fuel Type"}).drop(columns=["clean_description", "mainfuel"]) + + mds["Existing Fuel Type"].value_counts() mds_output_by_scenario = {} for scenario_id in scenario_ids: @@ -264,8 +311,9 @@ class Outputs: # Round Post SAP down to the nearest integer scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x)) scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x)) - - # TODO: Post heat demand + scenario_mds["Heating Demand Kwh/m2/y"] = ( + scenario_mds["Existing Heating Demand Kwh/m2/y"] - scenario_mds["heat_demand"] + ) scenario_mds = scenario_mds.rename( columns={ @@ -275,9 +323,21 @@ class Outputs: } ) + mds_output_by_scenario[scenario_id] = scenario_mds + + # We now save them to s3 as excels + for scenario_id, scenario_mds in mds_output_by_scenario.items(): + save_excel_to_s3( + df=scenario_mds, + file_key=f"engine_outputs/{self.format}/{self.today}_scenario_id={scenario_id}.xlsx", + bucket_name="retrofit-data-dev" + ) + def export(self): """ This function will export the data in the required format """ if self.format == "mds": self.export_mds() + + raise NotImplementedError("Export format not implemented") diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 5f6791f2..feeced10 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -108,6 +108,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property { "property_id": property_id, "type": rec["type"], + "measure_type": rec["measure_type"], "description": rec["description"], "estimated_cost": rec["total"], "default": rec["default"], @@ -121,7 +122,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], "already_installed": rec["already_installed"], - "head_demand": rec["heat_demand"] + "heat_demand": rec["heat_demand"] } for rec in recommendations_to_upload ] diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index a1743436..1089dced 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -15,6 +15,7 @@ class Recommendation(Base): property_id = Column(BigInteger, ForeignKey(PropertyModel.id), nullable=False) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) type = Column(String, nullable=False) + measure_type = Column(String) description = Column(String, nullable=False) estimated_cost = Column(Float) default = Column(Boolean, nullable=False) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6b8c5bea..2dc03e60 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -126,8 +126,8 @@ def extract_portfolio_aggregation_data( pre_retrofit_co2 = p.data["co2-emissions-current"] post_retrofit_co2 = pre_retrofit_co2 - carbon_savings - pre_retrofit_energy_bill = p.current_energy_bill - post_retrofit_energy_bill = p.current_energy_bill - sum( + pre_retrofit_energy_bill = sum(p.current_energy_bill.values()) + post_retrofit_energy_bill = sum(p.current_energy_bill.values()) - sum( [r["energy_cost_savings"] for r in default_recommendations] ) From bf45a5f4fa3d2b344235ef7d6dc171ddabc76dc2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 16:10:32 +0100 Subject: [PATCH 24/29] minor --- .../AirSourceHeatPumpEfficiency.py | 94 ++++++++++--------- etl/air_source_heat_pump/app.py | 4 +- recommendations/HeatingRecommender.py | 10 +- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py index 044cc830..e4eeedaf 100644 --- a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py +++ b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py @@ -22,68 +22,78 @@ class AirSourceHeatPumpEfficiency: def create_dataset(self): logger.info("Creating solar photo supply dataset") - all_counts = [] + heating_data = [] for dir in tqdm(self.file_directories): filepath = dir / "certificates.csv" df = pd.read_csv(filepath, low_memory=False) - df = df[~pd.isnull(df["UPRN"])] - df["UPRN"] = df["UPRN"].astype(int).astype(str) + # df = df[~pd.isnull(df["UPRN"])] + # df["UPRN"] = df["UPRN"].astype(int).astype(str) # Take entries after SAP12 df["LODGEMENT_DATE"] = pd.to_datetime(df["LODGEMENT_DATE"]) df = df[df["LODGEMENT_DATE"] > EARLIEST_EPC_DATE] - df = df[ - ~df["TENURE"].isin( - [ - "unknown", - "Not defined - use in the case of a new dwelling for which the intended tenure in not known. " - "It is not to be used for an existing dwelling" - ] - ) - ] + # df = df[ + # ~df["TENURE"].isin( + # [ + # "unknown", + # "Not defined - use in the case of a new dwelling for which the intended tenure in not known. " + # "It is not to be used for an existing dwelling" + # ] + # ) + # ] # Take entries that contain an air source heat pump df = df[ - df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False) - ] + ( + # Air source heat pumps + (df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") & + (df["MAINHEATCONT_DESCRIPTION"] == "Time and temperature zone control") + ) | + ( + # High heat retention storage + df["MAINHEATCONT_DESCRIPTION"] == "Controls for high heat retention storage heaters" + ) + ] # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]: df = df[~pd.isnull(df[col])] - # Get the columns we're interested in - df = df[ - [ - "PROPERTY_TYPE", - "BUILT_FORM", - "MAINHEAT_DESCRIPTION", - "MAINHEAT_ENERGY_EFF", - "MAINHEATCONT_DESCRIPTION", - "MAINHEATC_ENERGY_EFF", - "MAIN_FUEL", - "HOTWATER_DESCRIPTION", - "HOT_WATER_ENERGY_EFF", - "MAINS_GAS_FLAG" - ] + + heating_data.append(df) + + # temp + # import pickle + # with open("heating_data - delete me.pkl", "wb") as f: + # pickle.dump(heating_data, f) + + heating_df = pd.concat(heating_data) + # Clean construction age band + from etl.epc.DataProcessor import EPCDataProcessor + heating_df["CONSTRUCTION_AGE_BAND_CLEAN"] = heating_df["CONSTRUCTION_AGE_BAND"].apply( + lambda x: EPCDataProcessor.clean_construction_age_band(x) + ) + + ashp_df = heating_df[ + (heating_df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") & + # ~heating_df["CONSTRUCTION_AGE_BAND"].str.contains("England and Wales") + (~heating_df["CONSTRUCTION_AGE_BAND"].isin(["NO DATA!", "INVALID!"])) & + (heating_df["LODGEMENT_DATE"] >= pd.to_datetime("2019-01-01")) ] - - counts = df.groupby( + ashp_efficiencies = ( + ashp_df.groupby( [ - "PROPERTY_TYPE", - "BUILT_FORM", - "MAINHEAT_DESCRIPTION", + "CONSTRUCTION_AGE_BAND_CLEAN", + # "WALLS_DESCRIPTION", + # "ROOF_DESCRIPTION", "MAINHEAT_ENERGY_EFF", - "MAINHEATCONT_DESCRIPTION", - "MAINHEATC_ENERGY_EFF", - "MAIN_FUEL", - "HOTWATER_DESCRIPTION", - "HOT_WATER_ENERGY_EFF", - "MAINS_GAS_FLAG" ] - ).size().reset_index(name="count") + )["LMK_KEY"].count().reset_index() + ) - all_counts.append(counts) + ashp_df["MAINHEAT_ENERGY_EFF"].value_counts() - all_counts = pd.concat(all_counts) + ashp_efficiencies["CONSTRUCTION_AGE_BAND_CLEAN"].value_counts() + ashp_efficiency_agg all_counts_agg = all_counts.groupby( [ diff --git a/etl/air_source_heat_pump/app.py b/etl/air_source_heat_pump/app.py index ac87b34b..ed846d23 100644 --- a/etl/air_source_heat_pump/app.py +++ b/etl/air_source_heat_pump/app.py @@ -1,8 +1,10 @@ +import inspect from pathlib import Path from backend.app.plan.utils import get_cleaned from etl.air_source_heat_pump.AirSourceHeatPumpEfficiency import AirSourceHeatPumpEfficiency -DATA_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates" +file_src = inspect.getfile(lambda: None) +DATA_DIRECTORY = Path(file_src).parent / "local_data" / "all-domestic-certificates" def app(): diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index b4ca9a49..203bab87 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -434,7 +434,7 @@ class HeatingRecommender: ashp_costs_with_controls[key] += controls_rec[key] if controls_rec is None: - description = "Install an air source heat pump." + description = "Install a Mitsubish air source heat pump." elif already_installed: description = "The property already has an air source heat pump, no further action needed." else: @@ -457,8 +457,8 @@ class HeatingRecommender: ) simulation_config = { - "mainheat_energy_eff_ending": "Good", - "hot_water_energy_eff_ending": "Good" + "mainheat_energy_eff_ending": "Very Good", + "hot_water_energy_eff_ending": "Very Good" } description_simulation = { "mainheat-description": new_heating_description, @@ -725,7 +725,7 @@ class HeatingRecommender: description_prefix = "" controls_recommender.recommend( - heating_description="Electric storage heaters, radiators", description_prefix=description_prefix + heating_description="Electric storage heaters", description_prefix=description_prefix ) has_hhr = self.is_hhr_already_installed() @@ -740,7 +740,7 @@ class HeatingRecommender: self.property.main_heating["clean_description"] ]["hhr"]["mainheating_description"] else: - new_heating_description = "Electric storage heaters, radiators" + new_heating_description = "Electric storage heaters" # Set up artefacts, suitable for the simulation and regardless of controls heating_ending_config = MainHeatAttributes(new_heating_description).process() From aa97c777475cd036ee60004617cbb891f454f254 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 10:38:38 +0100 Subject: [PATCH 25/29] using elmhurst roof area methodology --- backend/Property.py | 17 ++------------ backend/apis/GoogleSolarApi.py | 1 - recommendations/recommendation_utils.py | 31 ++++++------------------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 491a886f..9108d40c 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -623,9 +623,7 @@ class Property: self.find_energy_sources() self.set_current_energy(kwh_client, kwh_predictions) - def set_solar_panel_configuration( - self, solar_panel_configuration, roof_area - ): + def set_solar_panel_configuration(self, solar_panel_configuration): """ This funtion inserts the solar panel configuration into the property object """ @@ -634,22 +632,11 @@ class Property: if not self.roof["is_flat"]: default_roof_area = estimate_pitched_roof_area( floor_area=self.insulation_floor_area, - floor_height=self.floor_height ) else: default_roof_area = self.insulation_floor_area - # Keep a record - self.roof_area_comparison = { - "api": roof_area, - "estimated": default_roof_area - } - - # We also set the roof area - if roof_area is None: - self.roof_area = default_roof_area - else: - self.roof_area = roof_area + self.roof_area = default_roof_area def set_current_energy(self, kwh_client, kwh_predictions): """ diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index bf67a786..606b6970 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -766,7 +766,6 @@ class GoogleSolarApi: "panel_performance": cls.default_panel_performance(property_instance=property_instance), "unit_share_of_energy": 1 }, - roof_area=None ) continue diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 3a0412b2..dcdd9c06 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -205,7 +205,7 @@ def get_wall_u_value( mapped_value = wall_uvalues_df[ wall_uvalues_df["Wall_type"] == mapped_description - ][age_band].values[0] + ][age_band].values[0] if pd.isnull(mapped_value) and "Park home" in mapped_description: # We don't know enough in this case so we default to 0 @@ -505,7 +505,7 @@ def get_floor_u_value( insulation_lookup = s11[ s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type - ] + ] if insulation_lookup.empty: insulation_thickness = 0 else: @@ -700,34 +700,17 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat): return int(string_thickness) -def estimate_pitched_roof_area(floor_area: float, floor_height: float) -> float: +def estimate_pitched_roof_area(floor_area: float) -> float: """ - This function will estimate the area of a pitched roof, given the floor area below the roof and the floor - height of the property. - - Given limited information about the home, this is a very rough method to estimate the roof area and we - assume the the room is a gable roof. - - We assume a roughly average pitch of 45 degrees - - Note that both floor area and height should be in the same units. E.g. if floor area is meters squared, - floor height should be in meters + This function mimics the methodology for calculating floor area in Elmhurst, so that we can simulate the outcomes + in a way that is consistent with the Elmhurst methodology. :param floor_area: area of the home's floor - :param floor_height: height of the home's floors :return: Numerical estimate of the surface area of the top of the pitched roof """ - # We estimate the length of the wall by just modelling the house as a square - wall_width = np.sqrt(floor_area) - - # We're modelling the roof as two triangles where we know two of the three sides. - # The floor height makes up one side and half of the wall width makes up the other side - slope = np.sqrt(np.square(wall_width / 2) + np.square(floor_height)) - - area = 2 * (slope * wall_width) - - return area + scalar = 1.0571283428862048 + return scalar * (floor_area / np.cos(np.radians(30))) def estimate_windows( From 5f0c0e7726b08694cad7f128895eac3667444dcc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 10:58:18 +0100 Subject: [PATCH 26/29] updating floor area unit tests --- .../tests/test_recommendation_utils.py | 58 ++++++------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index c42655eb..24ea6482 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -359,60 +359,36 @@ def test_park_home(): ) == 0 -def test_esimtate_pitched_roof_area(): - roof_area1 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=2 +def test_estimate_pitched_roof_area(): + roof_area0 = recommendation_utils.estimate_pitched_roof_area( + floor_area=80, + ) + assert np.isclose(roof_area0, 97.65333333333334) + + roof_area1 = recommendation_utils.estimate_pitched_roof_area( + floor_area=100, ) - assert np.isclose(roof_area1, 107.70329614269008) + assert np.isclose(roof_area1, 122.06666666666666) - # As the floor height gets bigger, the area should get bigger - roof_area2 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=3 + roof_area2 = recommendation_utils.estimate_pitched_roof_area( + floor_area=45, ) - assert np.isclose(roof_area2, 116.61903789690601) + assert np.isclose(roof_area2, 54.93) - # As the floor area gets smaller, the area should get smaller - roof_area3 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=1 + roof_area3 = recommendation_utils.estimate_pitched_roof_area( + floor_area=60, ) - assert np.isclose(roof_area3, 101.9803902718557) + assert np.isclose(roof_area3, 73.24) - # As the floor area decreases, area should decrease - roof_area4 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=50, floor_height=2 - ) - - assert np.isclose(roof_area4, 57.44562646538029) - - # As the floor area increases, area should increase - roof_area5 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=150, floor_height=2 - ) - - assert np.isclose(roof_area5, 157.797338380595) - - zero_roof_area = recommendation_utils.esimtate_pitched_roof_area( - floor_area=0, floor_height=1000 + zero_roof_area = recommendation_utils.estimate_pitched_roof_area( + floor_area=0, ) assert zero_roof_area == 0 - # If the floor height zero, we don't have a traingle, it's a flat roof - flat_roof_area = recommendation_utils.esimtate_pitched_roof_area( - floor_area=1000, floor_height=0 - ) - - assert flat_roof_area == 1000 - - zero_roof_area2 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=0, floor_height=0 - ) - - assert zero_roof_area2 == 0 - def test_external_wall_area(): # Arrange: Define the test cases From 92fce7f093fbbe83ad7890e46c3de872c234f977 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:03:52 +0100 Subject: [PATCH 27/29] Updating ligtiung unit test to include measure_type --- backend/Property.py | 2 +- .../tests/test_lighting_recommendations.py | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 9108d40c..be5479c5 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -112,7 +112,7 @@ class Property: self.measures = ast.literal_eval(measures) if measures else None self.uprn = epc_record.get("uprn") - self.uprn_source = self.data["uprn-source"] + self.uprn_source = self.data.get("uprn-source") self.full_sap_epc = epc_record.get("full_sap_epc") self.in_conservation_area, self.is_listed, self.is_heritage = None, None, None diff --git a/recommendations/tests/test_lighting_recommendations.py b/recommendations/tests/test_lighting_recommendations.py index 96440c01..32d607de 100644 --- a/recommendations/tests/test_lighting_recommendations.py +++ b/recommendations/tests/test_lighting_recommendations.py @@ -41,12 +41,18 @@ class TestLightingRecommendations: assert len(lr.recommendation) == 1 assert lr.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'low_energy_lighting', - 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, 'new_u_value': None, - 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, - 'description_simulation': {'lighting-energy-eff': 'Very Good', - 'lighting-description': 'Low energy lighting in all fixed outlets', - 'low-energy-lighting': 100}, 'total': 240.24, 'subtotal': 200.20000000000002, - 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, - 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False} + { + 'phase': 0, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting', + 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, + 'new_u_value': None, + 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, + 'description_simulation': { + 'lighting-energy-eff': 'Very Good', + 'lighting-description': 'Low energy lighting in all fixed outlets', + 'low-energy-lighting': 100 + }, + 'total': 240.24, 'subtotal': 200.20000000000002, + 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, + 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False + } ] From 8ff29603c326bc2b5ef2f472b79b5daa945e7ba9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:05:36 +0100 Subject: [PATCH 28/29] adding measure type to solar unit tests --- .../tests/test_solar_pv_recommendations.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index 05349f9c..a18291e5 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -3,12 +3,6 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations from backend.Property import Property from etl.epc.Record import EPCRecord import pandas as pd -from datetime import datetime -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 -from etl.solar.SolarPhotoSupply import SolarPhotoSupply -from recommendations.Recommendations import Recommendations -from backend.ml_models.api import ModelApi -import msgpack class TestSolarPvRecommendations: @@ -86,9 +80,10 @@ class TestSolarPvRecommendations: def test_valid_all_conditions(self, property_instance_valid_all): solar_pv = SolarPvRecommendations(property_instance_valid_all) solar_pv.recommend(phase=0) + assert len(solar_pv.recommendation) == 2 assert solar_pv.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'solar_pv', + 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' 'roof.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, @@ -97,17 +92,13 @@ class TestSolarPvRecommendations: 'description_simulation': {'photo-supply': 50.0} }, { - 'phase': 0, 'parts': [], 'type': 'solar_pv', - 'description': 'Install a 4.0 kilowatt-peak (kWp) ' - 'solar photovoltaic (PV) panel system ' - 'on 50% the roof, with a battery ' - 'storage system.', - 'starting_u_value': None, 'new_u_value': None, - 'sap_points': None, 'already_installed': False, - 'total': 7550.0, 'subtotal': 6291.666666666667, - 'vat': 1258.333333333333, 'labour_hours': 48, - 'labour_days': 2, 'photo_supply': 50.0, - 'has_battery': True, 'initial_ac_kwh_per_year': 3800, + 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' + 'roof, ' + 'with a battery storage system.', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7550.0, 'subtotal': 6291.666666666667, 'vat': 1258.333333333333, 'labour_hours': 48, + 'labour_days': 2, 'photo_supply': 50.0, 'has_battery': True, 'initial_ac_kwh_per_year': 3800, 'description_simulation': {'photo-supply': 50.0} } ] From 7bf897ceb4ce4e703bdb89bda58c2823dc0e1e21 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:17:11 +0100 Subject: [PATCH 29/29] fixed unitt tests --- .../tests/test_window_recommendations.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 0e36d105..ae6c6377 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -43,7 +43,7 @@ class TestWindowRecommendations: assert recommender.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing", 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, @@ -92,7 +92,7 @@ class TestWindowRecommendations: assert recommender2.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing", 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0, 'labour_hours': 0.0, @@ -193,7 +193,7 @@ class TestWindowRecommendations: assert recommender5.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing', 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, @@ -240,7 +240,7 @@ class TestWindowRecommendations: assert recommender6.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing', 'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to ' 'herigate building status', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, @@ -396,7 +396,7 @@ class TestWindowRecommendations: assert recommender9.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'double_glazing', 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, @@ -638,7 +638,8 @@ class TestWindowRecommendations: 'glazing_coverage_ending': 'full', 'id': '1+1' } - assert simulated_data[0] == expected_simulated_outcome + # Make sure all keys are the same, apart from days_to_ending + assert all([v == expected_simulated_outcome[k] for k, v in simulated_data[0].items() if k != "days_to_ending"]) # has_glazing_ending and glazing_coverage_ending are not in the starting record - test for this in case it # changes @@ -648,7 +649,7 @@ class TestWindowRecommendations: # Check which keys are different different = [] for k in simulated_data[0].keys(): - if k in ["id", 'has_glazing_ending', 'glazing_coverage_ending']: + if k in ["id", 'has_glazing_ending', 'glazing_coverage_ending', 'days_to_ending']: continue if simulated_data[0][k] != starting_record[k]: different.append( @@ -666,7 +667,6 @@ class TestWindowRecommendations: 'simulated': 'double glazing installed during or after 2002'}, {'variable': 'multi_glaze_proportion_ending', 'starting': 0.0, 'simulated': 100}, {'variable': 'windows_energy_eff_ending', 'starting': 'Very Poor', 'simulated': 'Average'}, - {'variable': 'days_to_ending', 'starting': 3642, 'simulated': 3713} ] assert different == expected_different