From cf7627a8d7fa06df445faf7637e06eefd7f8764b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 12:04:17 +0100 Subject: [PATCH 01/44] started setting up asset list and gathering council tax bands --- etl/customers/immo/pilot/asset_list.py | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 etl/customers/immo/pilot/asset_list.py diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py new file mode 100644 index 00000000..33f79729 --- /dev/null +++ b/etl/customers/immo/pilot/asset_list.py @@ -0,0 +1,44 @@ +import os + +import pandas as pd +from tqdm import tqdm + +from dotenv import load_dotenv +from utils.s3 import read_excel_from_s3 +from backend.SearchEpc import SearchEpc +from epc_api.client import EpcClient +from utils.s3 import save_csv_to_s3 + +# Read in the .env file in backend +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + +USER_ID = 8 +PORTFOLIO_ID = 70 + +council_tax_bands = [ + {'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'band': 'A'}, + {'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'band': 'A'}, + {'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'band': 'A'}, + {'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'band': 'A'}, + {'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'band': 'A'}, + {'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'band': 'B'}, + {'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'band': 'B'}, + {'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'band': 'C'}, + {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'band': 'A'}, + {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'band': 'A'}, +] + + +def app(): + raw_asset_list = read_excel_from_s3( + bucket_name="retrofit-datalake-dev", + file_key="customers/Immo/IMMO Sample Assets_Dudley.xlsx", + header_row=0 + ) + raw_asset_list = raw_asset_list.drop(columns=["Unnamed: 0"]) + # Extract address and postcode + raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0] + raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip() + + raw_asset_list[["address", "postcode"]].to_dict("records") From b791ecb054f0e5be39f91f78771f74ed80fe904d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 12:08:47 +0100 Subject: [PATCH 02/44] set up asset list --- etl/customers/immo/pilot/asset_list.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 33f79729..269ffe00 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -41,4 +41,23 @@ def app(): raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0] raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip() - raw_asset_list[["address", "postcode"]].to_dict("records") + council_tax_bands = pd.DataFrame(council_tax_bands) + asset_list = raw_asset_list.merge(council_tax_bands, how="left", on=["address", "postcode"]) + + # Store the data in s3 + filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(PORTFOLIO_ID), + "housing_type": "Private", + "goal": "Increase EPC", + "goal_value": "A", + "trigger_file_path": filename, + "budget": None, + } + print(body) From 5079170a25066e4ed3ab96c7a5034f1ddce5ada2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 13:34:15 +0100 Subject: [PATCH 03/44] pulled valuations for immo pilot from Zoopla --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 10 ++++++++++ backend/ml_models/Valuation.py | 11 +++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 50b8a837..c71533fa 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -142,6 +142,16 @@ async def trigger_plan(body: PlanTriggerRequest): ) ) + z = [] + for p in input_properties: + z.append( + { + "uprn": p.uprn, + "address": p.address, + "postcode": p.postcode, + } + ) + if not input_properties: return Response(status_code=204) diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 2bb7de32..251c016a 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -52,6 +52,17 @@ class PropertyValuation: 10070056829: 76_000, 10070056920: 76_000, 10023345463: 76_000, + # IMMO Dudley Pilot - search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/ + 90070461: 172_000, # Based on Zoopla + 90022227: 181_000, # Based on Zoopla + 90106884: 180_000, # Based on Zoopla + 90051858: 201_000, # Based on Zoopla + 90060989: 172_000, # Based on Zoopla + 90048026: 196_000, # Based on Zoopla + 90077535: 192_000, # Based on Zoopla + 90093693: 279_000, # Based on Zoopla + 90055152: 149_000, # Based on Zoopla + 90028499: 238_000, # Based on Zoopla } # We base our valuation uplifts on a number of sources From 5ac5cd7737a5b632258d130ea0e36057c25b0b6a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 14:02:19 +0100 Subject: [PATCH 04/44] fixing bug when setting phase for heating controls, without a recommendation --- backend/app/plan/router.py | 10 ---------- recommendations/HeatingRecommender.py | 7 ++++++- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index c71533fa..50b8a837 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -142,16 +142,6 @@ async def trigger_plan(body: PlanTriggerRequest): ) ) - z = [] - for p in input_properties: - z.append( - { - "uprn": p.uprn, - "address": p.address, - "postcode": p.postcode, - } - ) - if not input_properties: return Response(status_code=204) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index aec1f419..91730053 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -361,7 +361,12 @@ class HeatingRecommender: self.recommendations = combined_recommendations else: # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade - recommendation_phase += 1 + # but we'll only upgrade if we have a heating recommendation + has_heating_recommendation = any( + recommendation["type"] == "heating" for recommendation in self.recommendations + ) + if has_heating_recommendation: + recommendation_phase += 1 # The heating controls recommendation is distrinct from the boiler upgrade recommendation # We insert phase into the recommendations for heating controls for recommendation in controls_recommender.recommendation: From 4e4199345511c2aa8e838581cebe9e7c307c1475 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 17:20:49 +0100 Subject: [PATCH 05/44] savings --- etl/customers/immo/pilot/asset_list.py | 13 +------------ recommendations/optimiser/optimiser_functions.py | 6 +----- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 269ffe00..7939a555 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -1,18 +1,7 @@ -import os - import pandas as pd -from tqdm import tqdm - -from dotenv import load_dotenv from utils.s3 import read_excel_from_s3 -from backend.SearchEpc import SearchEpc -from epc_api.client import EpcClient from utils.s3 import save_csv_to_s3 -# Read in the .env file in backend -load_dotenv(dotenv_path="backend/.env") -EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") - USER_ID = 8 PORTFOLIO_ID = 70 @@ -28,6 +17,7 @@ council_tax_bands = [ {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'band': 'A'}, {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'band': 'A'}, ] +council_tax_bands = pd.DataFrame(council_tax_bands) def app(): @@ -41,7 +31,6 @@ def app(): raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0] raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip() - council_tax_bands = pd.DataFrame(council_tax_bands) asset_list = raw_asset_list.merge(council_tax_bands, how="left", on=["address", "postcode"]) # Store the data in s3 diff --git a/recommendations/optimiser/optimiser_functions.py b/recommendations/optimiser/optimiser_functions.py index 27838d6e..9860c5ea 100644 --- a/recommendations/optimiser/optimiser_functions.py +++ b/recommendations/optimiser/optimiser_functions.py @@ -20,10 +20,6 @@ def prepare_input_measures(property_recommendations, goal, housing_type): if not goal_key: raise NotImplementedError("Not implemented this gain type - investigate me") - # We don't include suspended and solid floor insulation as possible measures in private housing, because - # of the need to decant the tenant - ignored_measures = ["suspended_floor_insulation", "solid_floor_insulation"] if housing_type == "Private" else [] - input_measures = [] for recs in property_recommendations: input_measures.append( @@ -34,7 +30,7 @@ def prepare_input_measures(property_recommendations, goal, housing_type): "gain": rec[goal_key], "type": rec["type"] } - for rec in recs if rec["type"] not in ignored_measures + for rec in recs ] ) From 346b798c192e4c071640123379c021373d965543 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 17:26:00 +0100 Subject: [PATCH 06/44] removed whitespace --- backend/app/plan/router.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 50b8a837..bbf9261b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -75,7 +75,6 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Connecting to db") session = sessionmaker(bind=db_engine)() created_at = datetime.now().isoformat() - # TODO: We should store the trigger file path in the database with the plan so we can track the file that # triggered the plan From e0e60f8c9822aec63e1acb74bdb037a8a4840210 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 17:26:23 +0100 Subject: [PATCH 07/44] added whitespace --- backend/app/plan/router.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index bbf9261b..4b4d45e7 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -75,6 +75,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Connecting to db") session = sessionmaker(bind=db_engine)() created_at = datetime.now().isoformat() + # TODO: We should store the trigger file path in the database with the plan so we can track the file that # triggered the plan From 505fe0736becf7ad649d24ff68bf902825239b02 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 18:46:51 +0100 Subject: [PATCH 08/44] Updating optimiser to only optimise solar recommendations that include the battery --- backend/app/plan/router.py | 7 ++----- recommendations/SolarPvRecommendations.py | 3 ++- recommendations/optimiser/optimiser_functions.py | 12 +++++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4b4d45e7..6f179c79 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -75,7 +75,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Connecting to db") session = sessionmaker(bind=db_engine)() created_at = datetime.now().isoformat() - + # TODO: We should store the trigger file path in the database with the plan so we can track the file that # triggered the plan @@ -242,7 +242,7 @@ async def trigger_plan(body: PlanTriggerRequest): expected_adjusted_energy=expected_adjusted_energy ) - input_measures = prepare_input_measures(recommendations_with_impact, body.goal, body.housing_type) + input_measures = prepare_input_measures(recommendations_with_impact, body.goal) current_sap_points = int(property_instance.data["current-energy-efficiency"]) target_sap_points = epc_to_sap_lower_bound(body.goal_value) @@ -279,9 +279,6 @@ async def trigger_plan(body: PlanTriggerRequest): if ventilation_rec: selected_recommendations.add(ventilation_rec["recommendation_id"]) - # We check if the selected recommendation is wall ventilation and if so, we make sure - # mechanical ventilation is selected - # We'll use the set of selected recommendations to filter the recommendations to upload final_recommendations = [ [ diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 4cf1c1fc..f75003ce 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -122,6 +122,7 @@ class SolarPvRecommendations: **cost_result, # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale # back up here - "photo_supply": 100 * roof_coverage + "photo_supply": 100 * roof_coverage, + "has_battery": has_battery } ) diff --git a/recommendations/optimiser/optimiser_functions.py b/recommendations/optimiser/optimiser_functions.py index 9860c5ea..6159b930 100644 --- a/recommendations/optimiser/optimiser_functions.py +++ b/recommendations/optimiser/optimiser_functions.py @@ -1,17 +1,13 @@ -def prepare_input_measures(property_recommendations, goal, housing_type): +def prepare_input_measures(property_recommendations, goal): """ Basic function to convert recommendations_to_upload to a format that is suitable for the optimiser - large :param property_recommendations: object containing the recommendations, created in the plan trigger api :param goal: goal to be optimised for, should be one of the keys in gain_map. E.g. if the gain is SAP points, the goal should reflect that desired gain - :param housing_type: type of housing the recommendations are for - should be one of "Social" or "Private" :return: Nested list of input measures """ - if housing_type not in ["Social", "Private"]: - raise ValueError("Invalid housing type - investigate me") - goal_map = { "Increase EPC": "sap_points" } @@ -22,6 +18,12 @@ def prepare_input_measures(property_recommendations, goal, housing_type): input_measures = [] for recs in property_recommendations: + if recs[0]["type"] == "solar_pv": + # if the recommendation is a solar recommendation without a battery, we exclude it from the optimisation. + # That will ensure that the optimiser only considers solar recommendations with batteries, so we don't + # under-report the potential cost + recs = [r for r in recs if recs["has_battery"]] + input_measures.append( [ { From f04b79d6800fce396fdbc5494b66f221d43a9826 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 10 Apr 2024 18:54:58 +0100 Subject: [PATCH 09/44] fixed bug with selecting batter solar recommendations --- recommendations/optimiser/optimiser_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recommendations/optimiser/optimiser_functions.py b/recommendations/optimiser/optimiser_functions.py index 6159b930..d6353eea 100644 --- a/recommendations/optimiser/optimiser_functions.py +++ b/recommendations/optimiser/optimiser_functions.py @@ -22,7 +22,7 @@ def prepare_input_measures(property_recommendations, goal): # if the recommendation is a solar recommendation without a battery, we exclude it from the optimisation. # That will ensure that the optimiser only considers solar recommendations with batteries, so we don't # under-report the potential cost - recs = [r for r in recs if recs["has_battery"]] + recs = [r for r in recs if r["has_battery"]] input_measures.append( [ From 43af0de04732ba737459a1f04ccb50950287c235 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 10:30:36 +0100 Subject: [PATCH 10/44] Updated condittions we recommend loft insulation, so it is not recommended if the home has more than 200mm insulation in place already --- recommendations/RoofRecommendations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index eb1c6c4f..8d6a91e7 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -20,8 +20,9 @@ class RoofRecommendations: DIMINISHING_RETURNS_U_VALUE = 0.14 - # It is recommended that lofts should have at least 270mm of insulation - MINIMUM_LOFT_ISULATION_MM = 270 + # It is recommended that lofts should have at least 270mm of insulation. If the property has more than 200mm of + # loft insulation in place already, we do not recommend anything for the moment + MINIMUM_LOFT_ISULATION_MM = 200 # Flat roof should have at least 100mm of insulation MINIMUM_FLAT_ROOF_ISULATION_MM = 100 @@ -71,7 +72,7 @@ class RoofRecommendations: # Building regulations part L recommend installing at least 270mm of insulation, however generally we # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation # This only holds true for pitched roofs. - if (insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]: + if (insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]: return if (insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]: From db6fd58af4e89dcbdbecd436f2a9328ea6924521 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 10:56:17 +0100 Subject: [PATCH 11/44] changing the logic we use to recommend a combi boiler --- backend/Property.py | 13 +++++++++- recommendations/HeatingRecommender.py | 36 ++++++++++++++++++--------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index d3dd8395..6f2e648d 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -55,7 +55,13 @@ class Property: DATA_ANOMALY_MATCHES = DATA_ANOMALY_MATCHES - def __init__(self, id, postcode, address, epc_record): + # Surplus information, that can be provided as optional inputs, by a customer + n_bathrooms = None + n_bedrooms = None + + def __init__( + self, id, postcode, address, epc_record, **kwargs + ): self.epc_record = epc_record @@ -133,6 +139,11 @@ class Property: self.recommendations_scoring_data = [] + def parse_kwargs(self, kwargs): + # We extract the elements from kwargs that we recognise. Anything additional is ignored + self.n_bathrooms = kwargs.get("n_bathrooms", None) + self.n_bedrooms = kwargs.get("n_bedrooms", None) + def create_base_difference_epc_record(self, cleaned_lookup: dict): """ Creates a EPCDifferenceRecord object, which is used to store the difference between the current and diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 91730053..d4fe0a90 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -34,7 +34,6 @@ class HeatingRecommender: if has_electric_heating_description or no_heating_no_mains: # Recommend high heat retention storage heaters self.recommend_electric_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) - return # if the property has mains heating with boiler and radiators, we recommend optimal heating controls has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"] @@ -44,9 +43,16 @@ class HeatingRecommender: 'No system present, electric heaters assumed' ] and self.property.data["mains-gas-flag"] - if has_boiler or no_heating_has_mains: - self.recommend_boiler_upgrades(phase=phase, no_heating_has_mains=no_heating_has_mains) - return + # We also check if the property has electric heating, but it has access to the mains gas + electic_heating_has_mains = has_electric_heating_description and self.property.data["mains-gas-flag"] + + if has_boiler or no_heating_has_mains or electic_heating_has_mains: + # This indicates that the home previously did not have a boiler in place and so would require + # an overhaul to the system + system_change = not has_boiler + self.recommend_boiler_upgrades(phase=phase, system_change=system_change) + + return @staticmethod def check_simulation_difference(old_config, new_config): @@ -256,12 +262,14 @@ class HeatingRecommender: return closest_size - def recommend_boiler_upgrades(self, phase, no_heating_has_mains): + def recommend_boiler_upgrades(self, phase, system_change): """ This boiler recommendation will only recommend a like-for-like upgrade, since changing the system is generally more expensive :param phase: - :param no_heating_has_mains: indicaes if the property has no heating system, but has access to the mains gas + :param system_change: Indicates if the property would be undergoing a heating system change. This could be true + if the home didn't have a heating system in place, or if the home had electric heating + previously :return: """ @@ -279,17 +287,21 @@ class HeatingRecommender: num_heated_rooms=self.property.data["number-heated-rooms"], ) - # If heating and hot water come from the mains, we need a combi boiler, otherwise we need a regular boiler - hotwater_from_mains = self.property.hotwater["clean_description"] in ["From main system"] - - is_combi = hotwater_from_mains or no_heating_has_mains + # We recommend a combi boiler under the following conditions + # 1) If there are 4 or fewer rooms (we don't use heqted rooms because none of the rooms could be + # heated if there is no existing heating system). + # 2) There is more than 1 bathroom + is_combi = ( + (self.property.data["number-heated-rooms"] <= 4) or + (self.property.n_bathrooms not in [None, 0, 1]) + ) if is_combi: description = "Upgrade to a new combi boiler" else: - description = "Upgrade to a new boiler" + description = "Upgrade to a new gas condensing boiler" simulation_config = {"mainheat_energy_eff_ending": "Good"} - if no_heating_has_mains: + if system_change: # Installation of a boiler improves the hot water system so we need to reflect this in # the outcome of the recommendation heating_ending_config = MainHeatAttributes("Boiler and radiators, mains gas").process() From ac8cf271698788d4479626dae19f09a0027c79aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 11:20:29 +0100 Subject: [PATCH 12/44] created extract kwargs to read bathrooms and bedrooms --- backend/Property.py | 22 ++++++++++++++++++++++ backend/app/plan/router.py | 1 + 2 files changed, 23 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 6f2e648d..5fe9716e 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -139,6 +139,28 @@ class Property: self.recommendations_scoring_data = [] + @classmethod + def extract_kwargs(cls, kwargs): + """ + This method is to be used in the router, to extract the kwargs from the request and prevent any errors such as + non-integer values, or inputs that clash with the __init__ method of this class + :param kwargs: + :return: + """ + n_bathrooms = kwargs.get("n_bathrooms", None) + if n_bathrooms is not None: + # We add on a small value to ensure that the number of bathrooms is rounded up, in case the value is 0.5 + n_bathrooms = int(round(n_bathrooms + 1e-5)) + + n_bedrooms = kwargs.get("n_bedrooms", None) + if n_bedrooms is not None: + n_bedrooms = int(round(n_bedrooms + 1e-5)) + + return { + "n_bathrooms": n_bathrooms, + "n_bedrooms": n_bedrooms, + } + def parse_kwargs(self, kwargs): # We extract the elements from kwargs that we recognise. Anything additional is ignored self.n_bathrooms = kwargs.get("n_bathrooms", None) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6f179c79..7dc11bb9 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -139,6 +139,7 @@ async def trigger_plan(body: PlanTriggerRequest): address=epc_searcher.address_clean, postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, + **Property.extract_kwargs(config) ) ) From 2aa2e5947e6d29acf5c82962788a18ad9daf3351 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 11:36:14 +0100 Subject: [PATCH 13/44] adding bedrooms and bathrooms to asset list for immo --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- etl/customers/immo/pilot/asset_list.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 7939a555..9756e00b 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -33,6 +33,14 @@ def app(): asset_list = raw_asset_list.merge(council_tax_bands, how="left", on=["address", "postcode"]) + # We're provided with number of bathrooms and number of bedrooms. + asset_list = asset_list.rename( + columns={ + "No. of Beds": "n_bedrooms", + "No. of WC's": "n_bathrooms" + } + ) + # Store the data in s3 filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv" save_csv_to_s3( From 606fd3a615e2188f78e2721aef9732e5d0d76328 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 11:49:48 +0100 Subject: [PATCH 14/44] Adding parsing of kwargs to Property class --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/Property.py | 6 ++++-- backend/app/plan/router.py | 20 ++++++++++---------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/Property.py b/backend/Property.py index 5fe9716e..950c1ac9 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -139,6 +139,8 @@ class Property: self.recommendations_scoring_data = [] + self.parse_kwargs(kwargs) + @classmethod def extract_kwargs(cls, kwargs): """ @@ -150,11 +152,11 @@ class Property: n_bathrooms = kwargs.get("n_bathrooms", None) if n_bathrooms is not None: # We add on a small value to ensure that the number of bathrooms is rounded up, in case the value is 0.5 - n_bathrooms = int(round(n_bathrooms + 1e-5)) + n_bathrooms = int(round(float(n_bathrooms) + 1e-5)) n_bedrooms = kwargs.get("n_bedrooms", None) if n_bedrooms is not None: - n_bedrooms = int(round(n_bedrooms + 1e-5)) + n_bedrooms = int(round(float(n_bedrooms) + 1e-5)) return { "n_bathrooms": n_bathrooms, diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 7dc11bb9..3cb2027d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -109,16 +109,16 @@ async def trigger_plan(body: PlanTriggerRequest): session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn ) # if a new record was not created, we don't produduce recommendations - if not is_new: - continue - - create_property_targets( - session, - property_id=property_id, - portfolio_id=body.portfolio_id, - epc_target=body.goal_value, - heat_demand_target=None - ) + # if not is_new: + # continue + # + # create_property_targets( + # session, + # property_id=property_id, + # portfolio_id=body.portfolio_id, + # epc_target=body.goal_value, + # heat_demand_target=None + # ) epc_records = { 'original_epc': epc_searcher.newest_epc.copy(), From 69424149510c38f59d1d847cbcef740a287da23b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 15:40:50 +0100 Subject: [PATCH 15/44] Updating heating recommender to recommend heating controls, with the heating change --- backend/app/plan/router.py | 21 ++++++++++----------- recommendations/HeatingRecommender.py | 6 +++--- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3cb2027d..4b91566e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -108,17 +108,16 @@ async def trigger_plan(body: PlanTriggerRequest): property_id, is_new = create_property( session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn ) - # if a new record was not created, we don't produduce recommendations - # if not is_new: - # continue - # - # create_property_targets( - # session, - # property_id=property_id, - # portfolio_id=body.portfolio_id, - # epc_target=body.goal_value, - # heat_demand_target=None - # ) + if not is_new: + continue + + create_property_targets( + session, + property_id=property_id, + portfolio_id=body.portfolio_id, + epc_target=body.goal_value, + heat_demand_target=None + ) epc_records = { 'original_epc': epc_searcher.newest_epc.copy(), diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index d4fe0a90..6e4b2230 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -48,7 +48,7 @@ class HeatingRecommender: if has_boiler or no_heating_has_mains or electic_heating_has_mains: # This indicates that the home previously did not have a boiler in place and so would require - # an overhaul to the system + # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler system_change = not has_boiler self.recommend_boiler_upgrades(phase=phase, system_change=system_change) @@ -353,8 +353,8 @@ class HeatingRecommender: if not controls_recommender.recommendation: return - if no_heating_has_mains: - # We combine the heating and controls recommendations + if system_change: + # We combine the heating and controls recommendations, in the case of a system change boiler_recommendation = self.recommendations[0].copy() combined_recommendations = [] for controls_recommendation in controls_recommender.recommendation: From 014d51c0605e853351b621fbeafdf8ca3b870cbf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 16:09:31 +0100 Subject: [PATCH 16/44] fixing the case where we recommend a boiler and new heating controls, as well as an improved electrical system --- recommendations/HeatingRecommender.py | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 6e4b2230..1813e5e8 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -278,6 +278,7 @@ class HeatingRecommender: # We now recommend boiler upgrades, if applicable simulation_config = {} boiler_costs = {} + boiler_recommendation = {} if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]: boiler_size = self.estimate_boiler_size( property_type=self.property.data["property-type"], @@ -290,10 +291,12 @@ class HeatingRecommender: # We recommend a combi boiler under the following conditions # 1) If there are 4 or fewer rooms (we don't use heqted rooms because none of the rooms could be # heated if there is no existing heating system). - # 2) There is more than 1 bathroom + # 2) There 1 or fewer bathrooms + # Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple + # bathrooms is_combi = ( (self.property.data["number-heated-rooms"] <= 4) or - (self.property.n_bathrooms not in [None, 0, 1]) + (self.property.n_bathrooms in [None, 0, 1]) ) if is_combi: description = "Upgrade to a new combi boiler" @@ -328,21 +331,19 @@ class HeatingRecommender: boiler_costs = self.costs.low_carbon_boiler(is_combi=is_combi, size=f"{boiler_size}kw") - self.recommendations.append( - { - "phase": recommendation_phase, - "parts": [ - # TODO - ], - "type": "heating", - "description": description, - "starting_u_value": None, - "new_u_value": None, - "sap_points": None, - "simulation_config": simulation_config, - **boiler_costs - } - ) + boiler_recommendation = { + "phase": recommendation_phase, + "parts": [ + # TODO + ], + "type": "heating", + "description": description, + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + "simulation_config": simulation_config, + **boiler_costs + } # We recommend the heating controls # If the property did not previously have a boiler, we combine @@ -355,7 +356,6 @@ class HeatingRecommender: if system_change: # We combine the heating and controls recommendations, in the case of a system change - boiler_recommendation = self.recommendations[0].copy() combined_recommendations = [] for controls_recommendation in controls_recommender.recommendation: combined_recommendation = self.combine_heating_and_controls( From 88f43bcc822b4550540c88e7363d920937563072 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 16:49:44 +0100 Subject: [PATCH 17/44] fixed the combi boiler logic --- recommendations/HeatingControlRecommender.py | 3 ++- recommendations/HeatingRecommender.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 95b5e3b1..76eaba4f 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -215,7 +215,8 @@ class HeatingControlRecommender: { "type": "heating_control", "parts": [], - "description": "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves", + "description": "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves " + "(time & temperature zone control)", **self.costs.time_and_temperature_zone_control( number_heated_rooms=int(self.property.data["number-heated-rooms"]) ), diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 1813e5e8..bd4d87a2 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -295,7 +295,7 @@ class HeatingRecommender: # Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple # bathrooms is_combi = ( - (self.property.data["number-heated-rooms"] <= 4) or + (self.property.data["number-heated-rooms"] <= 4) and (self.property.n_bathrooms in [None, 0, 1]) ) if is_combi: @@ -370,7 +370,7 @@ class HeatingRecommender: combined_recommendations.extend(combined_recommendation) # Overwrite the existing boiler recommendation - self.recommendations = combined_recommendations + self.recommendations.extend(combined_recommendations) else: # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade # but we'll only upgrade if we have a heating recommendation From 61584a6320bfd50bb4f18266a09cc1bb1e4e2ba1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 18:14:38 +0100 Subject: [PATCH 18/44] extend recommendations to cover portable electric heaters --- recommendations/Costs.py | 18 ++++++++++++- recommendations/HeatingRecommender.py | 37 ++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index e5ceb0c0..f4ac259b 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -79,6 +79,10 @@ CONVENTIONAL_BOILER_COSTS = { "40kw": 1776 } +# Assumes 3 hours to remove each heater (including re-decorating) +ROOM_HEATER_REMOVAL_COST = 120 +ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3 + class Costs: """ @@ -1100,7 +1104,7 @@ class Costs: "labour_days": labour_days, } - def low_carbon_boiler(self, is_combi, size): + def boiler(self, is_combi, size, exising_room_heaters, n_heated_rooms): """ Based on a basic estimate of median value £2600 to install a low carbon combi boiler :return: @@ -1118,6 +1122,18 @@ class Costs: labour_cost = labour_rate * self.labour_adjustment_factor * labour_days # Add contingency and preliminaries labour_cost = labour_cost * (1 + self.CONTINGENCY + self.PRELIMINARIES) + + # if there are existing room heaters, we need to add the cost of removing them + if exising_room_heaters: + removal_cost = ROOM_HEATER_REMOVAL_COST * n_heated_rooms + removal_labour_hours = ROOM_HEATER_REMOVAL_LABOUR_HOURS * n_heated_rooms + else: + removal_cost = 0 + removal_labour_hours = 0 + + labour_cost = labour_cost + removal_cost + labour_days = labour_days + (removal_labour_hours / 8) + vat = labour_cost * self.VAT_RATE subtotal_before_vat = unit_cost + labour_cost diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index bd4d87a2..14509eea 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -43,14 +43,36 @@ class HeatingRecommender: 'No system present, electric heaters assumed' ] and self.property.data["mains-gas-flag"] + has_gas_heaters = ( + self.property.main_heating["clean_description"] in ["Room heaters, mains gas"] and + self.property.data["mains-gas-flag"] + ) + # We also check if the property has electric heating, but it has access to the mains gas electic_heating_has_mains = has_electric_heating_description and self.property.data["mains-gas-flag"] - if has_boiler or no_heating_has_mains or electic_heating_has_mains: + portable_heaters_has_mains = ( + self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"] and + self.property.data["mains-gas-flag"] + ) + + if ( + has_boiler or + no_heating_has_mains or + electic_heating_has_mains or + has_gas_heaters or + portable_heaters_has_mains + ): # This indicates that the home previously did not have a boiler in place and so would require # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler system_change = not has_boiler - self.recommend_boiler_upgrades(phase=phase, system_change=system_change) + exising_room_heaters = self.property.main_heating["clean_description"] in [ + "Room heaters, electric", "Room heaters, mains gas" + ] + + self.recommend_boiler_upgrades( + phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters + ) return @@ -262,7 +284,7 @@ class HeatingRecommender: return closest_size - def recommend_boiler_upgrades(self, phase, system_change): + def recommend_boiler_upgrades(self, phase, system_change, exising_room_heaters): """ This boiler recommendation will only recommend a like-for-like upgrade, since changing the system is generally more expensive @@ -270,6 +292,8 @@ class HeatingRecommender: :param system_change: Indicates if the property would be undergoing a heating system change. This could be true if the home didn't have a heating system in place, or if the home had electric heating previously + :param exising_room_heaters: Indicates if the property had room heaters previously - if so, a boiler + recommendation will need to be accompanied by removal of the room heaters :return: """ @@ -329,7 +353,12 @@ class HeatingRecommender: "hot_water_energy_eff_ending": "Good" } - boiler_costs = self.costs.low_carbon_boiler(is_combi=is_combi, size=f"{boiler_size}kw") + boiler_costs = self.costs.boiler( + is_combi=is_combi, + size=f"{boiler_size}kw", + exising_room_heaters=exising_room_heaters, + n_heated_rooms=self.property.data["number-heated-rooms"] + ) boiler_recommendation = { "phase": recommendation_phase, From 3ecd7a974276bb6f4296124c6acf7e55f280e574 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 11 Apr 2024 19:14:49 +0100 Subject: [PATCH 19/44] added simulation for secondary heating --- backend/Property.py | 6 ++- recommendations/Costs.py | 45 ++++++++++++++++------ recommendations/HeatingRecommender.py | 2 +- recommendations/Recommendations.py | 8 ++++ recommendations/SecondaryHeating.py | 55 +++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 recommendations/SecondaryHeating.py diff --git a/backend/Property.py b/backend/Property.py index 950c1ac9..0f5e7e77 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -456,7 +456,9 @@ class Property: "double glazing installed during or after 2002" ) - if recommendation["type"] in ["heating", "hot_water_tank_insulation", "heating_control"]: + if recommendation["type"] in [ + "heating", "hot_water_tank_insulation", "heating_control", "secondary_heating" + ]: # We update the data, as defined in the recommendaton simulation_config = recommendation["simulation_config"] @@ -477,7 +479,7 @@ class Property: "loft_insulation", "room_roof_insulation", "flat_roof_insulation", "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation", "windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation", - "heating_control", + "heating_control", "secondary_heating" ]: raise NotImplementedError( "Implement me, given type %s" % recommendation["type"] diff --git a/recommendations/Costs.py b/recommendations/Costs.py index f4ac259b..45c17102 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1104,6 +1104,28 @@ class Costs: "labour_days": labour_days, } + def heater_removal(self, n_rooms): + """ + Estimates the costs of removal of heaters, including the redecoration costs of the space behind the heater + :return: + """ + + removal_cost = ROOM_HEATER_REMOVAL_COST * n_rooms + removal_labour_hours = ROOM_HEATER_REMOVAL_LABOUR_HOURS * n_rooms + + vat = removal_cost * self.VAT_RATE + + subtotal_before_vat = removal_cost + total_cost = subtotal_before_vat + vat + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat, + "labour_hours": removal_labour_hours, + "labour_days": np.ceil(removal_labour_hours / 8), + } + def boiler(self, is_combi, size, exising_room_heaters, n_heated_rooms): """ Based on a basic estimate of median value £2600 to install a low carbon combi boiler @@ -1114,6 +1136,7 @@ class Costs: # The unit cost is the cost without VAT # We now need to estimate the cost of the works labour_days = 2 + labour_hours = labour_days * 8 labour_rate = 500 # Average cost of installation is 1 (maybe 2days) at £300 per day @@ -1123,26 +1146,26 @@ class Costs: # Add contingency and preliminaries labour_cost = labour_cost * (1 + self.CONTINGENCY + self.PRELIMINARIES) - # if there are existing room heaters, we need to add the cost of removing them - if exising_room_heaters: - removal_cost = ROOM_HEATER_REMOVAL_COST * n_heated_rooms - removal_labour_hours = ROOM_HEATER_REMOVAL_LABOUR_HOURS * n_heated_rooms - else: - removal_cost = 0 - removal_labour_hours = 0 - - labour_cost = labour_cost + removal_cost - labour_days = labour_days + (removal_labour_hours / 8) + # labour_days = labour_days + (removal_labour_hours / 8) vat = labour_cost * self.VAT_RATE subtotal_before_vat = unit_cost + labour_cost total_cost = subtotal_before_vat + vat + # if there are existing room heaters, we need to add the cost of removing them + if exising_room_heaters: + removal_costing = self.heater_removal(n_rooms=n_heated_rooms) + # Add the totals to the existing totals + total_cost += removal_costing["total"] + subtotal_before_vat += removal_costing["subtotal"] + labour_hours += removal_costing["labour_hours"] + labour_days += removal_costing["labour_days"] + return { "total": total_cost, "subtotal": subtotal_before_vat, "vat": vat, - "labour_hours": labour_days * 8, + "labour_hours": labour_hours, "labour_days": labour_days, } diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 14509eea..92457a27 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -319,7 +319,7 @@ class HeatingRecommender: # Otherwise, we recommend a gas condensing boiler, which will server a larger property, that has multiple # bathrooms is_combi = ( - (self.property.data["number-heated-rooms"] <= 4) and + (self.property.number_of_rooms <= 4) and (self.property.n_bathrooms in [None, 0, 1]) ) if is_combi: diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 902023dc..68fead16 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -11,6 +11,7 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations from recommendations.WindowsRecommendations import WindowsRecommendations from recommendations.HeatingRecommender import HeatingRecommender from recommendations.HotwaterRecommendations import HotwaterRecommendations +from recommendations.SecondaryHeating import SecondaryHeating from backend.ml_models.AnnualBillSavings import AnnualBillSavings @@ -46,6 +47,7 @@ class Recommendations: self.solar_recommender = SolarPvRecommendations(property_instance=property_instance) self.heating_recommender = HeatingRecommender(property_instance=property_instance) self.hotwater_recommender = HotwaterRecommendations(property_instance=property_instance) + self.secondary_heating_recommender = SecondaryHeating(property_instance=property_instance) def recommend(self): @@ -130,6 +132,12 @@ class Recommendations: property_recommendations.append(self.lighting_recommender.recommendation) phase += 1 + if "secondary_heating" not in self.exclusions: + self.secondary_heating_recommender.recommend(phase=phase) + if self.secondary_heating_recommender.recommendation: + property_recommendations.append(self.secondary_heating_recommender.recommendation) + phase += 1 + # Renewables if "solar_pv" not in self.exclusions: self.solar_recommender.recommend(phase=phase) diff --git a/recommendations/SecondaryHeating.py b/recommendations/SecondaryHeating.py new file mode 100644 index 00000000..f31c4c05 --- /dev/null +++ b/recommendations/SecondaryHeating.py @@ -0,0 +1,55 @@ +from recommendations.Costs import Costs +from backend.Property import Property + + +class SecondaryHeating: + """ + This class recommends the removal of the secondary heating system for properties that have a primary heating + system. + """ + + # The list of existing heating systems that are accepted + ACCEPTED_MAINHEAT_DESCRIPTIONS = ["Boiler and radiators, mains gas"] + ACCEPTED_SECONDHEAT_DESCRIPTIONS = ["Room heaters, electric"] + # These are the heaters where works are required to remove them + FIXED_HEATER_DESCRIPTIONS = ["Room heaters, electric"] + + def __init__(self, property_instance: Property): + self.property = property_instance + self.costs = Costs(self.property) + + self.recommendation = [] + + def recommend(self, phase: int): + # Reset + self.recommendation = [] + + if self.property.main_heating["clean_description"] not in self.ACCEPTED_MAINHEAT_DESCRIPTIONS: + return + + # TODO: We need to clean secondary data + if self.property.data['secondheat-description'] not in self.ACCEPTED_SECONDHEAT_DESCRIPTIONS: + return + + if self.property.data['secondheat-description'] in self.FIXED_HEATER_DESCRIPTIONS: + # We have an associated cost otherwise, there is no cost + n_rooms = self.property.data['number-heated-rooms'] + else: + n_rooms = 0 + + costs = self.costs.heater_removal(n_rooms=n_rooms) + self.recommendation.append( + { + "phase": phase, + "parts": [], + "type": "secondary_heating", + "description": "Remove the secondary heating system", + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + **costs, + "simulation_config": { + "secondheat_description_ending": "None" + } + } + ) From 0b75ec9210e7c7c097bf4e6b5d2d87cb273af6cd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 14:41:19 +0100 Subject: [PATCH 20/44] Added patches and overrides to immo asset list --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- .../AirSourceHeatPumpEfficiency.py | 78 +++++++++++++++++++ etl/air_source_heat_pump/app.py | 24 ++++++ etl/customers/immo/pilot/asset_list.py | 70 ++++++++++++++++- 5 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py create mode 100644 etl/air_source_heat_pump/app.py diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py new file mode 100644 index 00000000..2ba82e77 --- /dev/null +++ b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py @@ -0,0 +1,78 @@ +import pandas as pd +from tqdm import tqdm +from utils.s3 import save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet +from utils.logger import setup_logger +from etl.epc.settings import EARLIEST_EPC_DATE + +logger = setup_logger() + + +class AirSourceHeatPumpEfficiency: + + def __init__(self, file_directories, cleaned_lookup): + """ + :param file_directories: A list of directories where files are stored. + :param cleaned_lookup: A dictionary containing cleaned lookup data. + """ + self.file_directories = file_directories + self.cleaned_lookup = cleaned_lookup + + self.results = [] + + def create_dataset(self): + logger.info("Creating solar photo supply dataset") + for dir in tqdm(self.file_directories): + filepath = dir / "certificates.csv" + df = pd.read_csv(filepath, low_memory=False) + df = df[~pd.isnull(df["UPRN"])] + df["UPRN"] = df["UPRN"].astype(int).astype(str) + # Take entries after SAP12 + df["LODGEMENT_DATE"] = pd.to_datetime(df["LODGEMENT_DATE"]) + df = df[df["LODGEMENT_DATE"] > EARLIEST_EPC_DATE] + + df = df[ + ~df["TENURE"].isin( + [ + "unknown", + "Not defined - use in the case of a new dwelling for which the intended tenure in not known. " + "It is not to be used for an existing dwelling" + ] + ) + ] + + # Take entries that contain an air source heat pump + df = df[ + df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False) + ] + # Get the columns we're interested in + df = df[ + [ + "MAINHEAT_DESCRIPTION", + "MAINHEAT_ENERGY_EFF", + "MAINHEATCONT_DESCRIPTION", + "MAINHEATC_ENERGY_EFF", + "MAIN_FUEL", + "HOTWATER_DESCRIPTION", + "HOT_WATER_ENERGY_EFF", + "MAINS_GAS_FLAG" + ] + ] + + counts = df.groupby( + [ + "MAINHEAT_DESCRIPTION", + "MAINHEAT_ENERGY_EFF", + "MAINHEATCONT_DESCRIPTION", + "MAINHEATC_ENERGY_EFF", + "MAIN_FUEL", + "HOTWATER_DESCRIPTION", + "HOT_WATER_ENERGY_EFF", + "MAINS_GAS_FLAG" + ] + ).size().reset_index(name="count") + + # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA + for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]: + df = df[~pd.isnull(df[col])] + # Take newest LODGEMENT_DATE per UPRN + df = df.sort_values(by="LODGEMENT_DATE", ascending=False).drop_duplicates(subset=["UPRN"]) diff --git a/etl/air_source_heat_pump/app.py b/etl/air_source_heat_pump/app.py new file mode 100644 index 00000000..ac87b34b --- /dev/null +++ b/etl/air_source_heat_pump/app.py @@ -0,0 +1,24 @@ +from pathlib import Path +from backend.app.plan.utils import get_cleaned +from etl.air_source_heat_pump.AirSourceHeatPumpEfficiency import AirSourceHeatPumpEfficiency + +DATA_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates" + + +def app(): + """ + This code reads in the EPC dataset and looks at the efficiency values for heating systems that inclue air source + heat pumps. This dataset is then used to inform the recommendations for the air source heat pump, so we know + how to set the simulation + :return: + """ + + directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()] + cleaned_lookup = get_cleaned() + + ashp_data_client = AirSourceHeatPumpEfficiency( + file_directories=directories, + cleaned_lookup=cleaned_lookup + ) + + ashp_data_client.create_dataset() diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 9756e00b..0da8f885 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -19,6 +19,40 @@ council_tax_bands = [ ] council_tax_bands = pd.DataFrame(council_tax_bands) +# This is information we need to override on the EPC itself, for instance if a new survey has been conducted and +# that has not reached the API +patches = [ + { + 'address': '6 Beech Road', 'postcode': 'DY1 4BP', + 'walls-description': 'Mixed: Filled cavity and external insulated solid brick', + 'walls-energy-eff': 'Good', + 'roof-description': 'Pitched, 12 mm loft insulation', + 'roof-energy-eff': 'Very Poor', + 'windows-description': 'Fully double glazed', + 'windows-energy-eff': 'Good', + 'mainheat-description': 'Room heaters, electric', + 'mainheat-energy-eff': 'Very Poor', + 'mainheatcont-description': 'Appliance thermostats', + 'mainheatc-energy-eff': 'Good', + 'lighting-description': 'Low energy lighting in 25% of fixed outlets', + 'lighting-energy-eff': 'Good', + 'floor-description': 'Mixed: Solid no insulation and suspended no insulation', + 'secondheat-description': 'None', + 'current-energy-efficiency': '32', + } +] + +# This is information that is found as a result of the non-invasives, that mean that certain measures +# have been installed already. To reflect this in the front end, it is included in the recommendation, however +# the cost is removed and instead, a message is presented saying that the measure is already installed. +overrides = [ + { + 'address': '5 Oaklands', + 'postcode': 'B62 0JA', + "overrides": ["windows_glazing"] + } +] + def app(): raw_asset_list = read_excel_from_s3( @@ -41,7 +75,7 @@ def app(): } ) - # Store the data in s3 + # Store the asset list in s3 filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv" save_csv_to_s3( dataframe=asset_list, @@ -49,12 +83,44 @@ def app(): file_name=filename ) + # Store overrides in s3 + overrides_filename = f"{USER_ID}/{PORTFOLIO_ID}/overrides.json" + save_csv_to_s3( + dataframe=pd.DataFrame(overrides), + bucket_name="retrofit-plan-inputs-dev", + file_name=overrides_filename + ) + + # Store patches in s3 + patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json" + save_csv_to_s3( + dataframe=pd.DataFrame(patches), + bucket_name="retrofit-plan-inputs-dev", + file_name=patches_filename + ) + + # EPC C portoflio body = { "portfolio_id": str(PORTFOLIO_ID), "housing_type": "Private", "goal": "Increase EPC", - "goal_value": "A", + "goal_value": "C", "trigger_file_path": filename, + "overrides_file_path": overrides_filename, + "patches_file_path": patches_filename, + "budget": None, + } + print(body) + + # EPC B portoflio + body = { + "portfolio_id": str(PORTFOLIO_ID + 1), + "housing_type": "Private", + "goal": "Increase EPC", + "goal_value": "B", + "trigger_file_path": filename, + "overrides_file_path": overrides_filename, + "patches_file_path": patches_filename, "budget": None, } print(body) From ab180f65225507c6d666516fd70259a7c0ec4ac5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:06:12 +0100 Subject: [PATCH 21/44] Added overrides and patches to router --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/Property.py | 6 ++++- backend/app/plan/router.py | 34 ++++++++++++++++++-------- backend/app/plan/schemas.py | 2 ++ etl/customers/immo/pilot/asset_list.py | 4 +-- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/Property.py b/backend/Property.py index 0f5e7e77..882e450c 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -60,7 +60,7 @@ class Property: n_bedrooms = None def __init__( - self, id, postcode, address, epc_record, **kwargs + self, id, postcode, address, epc_record, overrides=None, **kwargs ): self.epc_record = epc_record @@ -74,6 +74,10 @@ class Property: } self.old_data = epc_record.get("old_data") self.property_dimensions = None + # This is a list of measures that have already been installed in the property, typically found as a result + # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the + # cost and instead, provide a message that the measure has already been installed + self.overrides = overrides self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4b91566e..8d39c97f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -44,20 +44,15 @@ BATCH_SIZE = 5 SCORING_BATCH_SIZE = 400 -def patch_epc(config, epc_records): +def patch_epc(patch, epc_records): """ This utility function is useful to patch the epc data if we have data from the customer :return: """ - number_habitable_rooms = config.get("number-habitable-rooms", None) - number_heated_rooms = config.get("number-heated-rooms", None) - - if number_habitable_rooms is not None: - epc_records["original_epc"]["number-habitable-rooms"] = int(number_habitable_rooms) - - if number_heated_rooms is not None: - epc_records["original_epc"]["number-heated-rooms"] = int(number_heated_rooms) + for patch_variable, patch_value in patch.items(): + if patch_variable in epc_records["original_epc"]: + epc_records["original_epc"][patch_variable] = patch_value return epc_records @@ -85,6 +80,17 @@ async def trigger_plan(body: PlanTriggerRequest): session.begin() logger.info("Getting the inputs") plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path) + # If we have patches or overrides, we should read them in here + patches = [] + if body.patches_file_path: + patches = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.patches_file_path) + + overrides = [] + if body.overrides_file_path: + overrides = read_csv_from_s3( + bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.overrides_file_path + ) + cleaning_data = read_dataframe_from_s3_parquet( bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet", ) @@ -124,7 +130,11 @@ async def trigger_plan(body: PlanTriggerRequest): 'full_sap_epc': epc_searcher.full_sap_epc.copy(), 'old_data': epc_searcher.older_epcs.copy(), } - epc_records = patch_epc(config, epc_records) + + patch = next(( + x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + ), None) + epc_records = patch_epc(patch, epc_records) prepared_epc = EPCRecord( epc_records=epc_records, @@ -132,12 +142,16 @@ async def trigger_plan(body: PlanTriggerRequest): cleaning_data=cleaning_data ) + overrides = next(( + x for x in overrides if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + ), None) input_properties.append( Property( id=property_id, address=epc_searcher.address_clean, postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, + overrides=overrides, **Property.extract_kwargs(config) ) ) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index b8a99704..ec49e41e 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -9,6 +9,8 @@ class PlanTriggerRequest(BaseModel): goal_value: str portfolio_id: int trigger_file_path: str + overrides_file_path: Optional[str] = None + patches_file_path: Optional[str] = None exclusions: Optional[conlist(str, min_items=1)] = None # Pre-defined list of possibilities for exclusions diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 0da8f885..15681d42 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -24,7 +24,7 @@ council_tax_bands = pd.DataFrame(council_tax_bands) patches = [ { 'address': '6 Beech Road', 'postcode': 'DY1 4BP', - 'walls-description': 'Mixed: Filled cavity and external insulated solid brick', + 'walls-description': 'Cavity wall, filled cavity', 'walls-energy-eff': 'Good', 'roof-description': 'Pitched, 12 mm loft insulation', 'roof-energy-eff': 'Very Poor', @@ -36,7 +36,7 @@ patches = [ 'mainheatc-energy-eff': 'Good', 'lighting-description': 'Low energy lighting in 25% of fixed outlets', 'lighting-energy-eff': 'Good', - 'floor-description': 'Mixed: Solid no insulation and suspended no insulation', + 'floor-description': 'Solid, no insulation (assumed)', 'secondheat-description': 'None', 'current-energy-efficiency': '32', } From 8e2d823693f53ad47a4fe857fd8f24d84c0c4ec1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:11:51 +0100 Subject: [PATCH 22/44] corrected parsing of overrides --- backend/Property.py | 4 ++-- backend/app/plan/router.py | 8 ++++---- etl/customers/immo/pilot/asset_list.py | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 882e450c..3fac3667 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -60,7 +60,7 @@ class Property: n_bedrooms = None def __init__( - self, id, postcode, address, epc_record, overrides=None, **kwargs + self, id, postcode, address, epc_record, override=None, **kwargs ): self.epc_record = epc_record @@ -77,7 +77,7 @@ class Property: # This is a list of measures that have already been installed in the property, typically found as a result # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the # cost and instead, provide a message that the measure has already been installed - self.overrides = overrides + self.override = override self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8d39c97f..08ce0dcc 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -133,7 +133,7 @@ async def trigger_plan(body: PlanTriggerRequest): patch = next(( x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) - ), None) + ), {}) epc_records = patch_epc(patch, epc_records) prepared_epc = EPCRecord( @@ -142,16 +142,16 @@ async def trigger_plan(body: PlanTriggerRequest): cleaning_data=cleaning_data ) - overrides = next(( + override = next(( x for x in overrides if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) - ), None) + ), {}) input_properties.append( Property( id=property_id, address=epc_searcher.address_clean, postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, - overrides=overrides, + override=override, **Property.extract_kwargs(config) ) ) diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 15681d42..07ebe884 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -39,6 +39,9 @@ patches = [ 'floor-description': 'Solid, no insulation (assumed)', 'secondheat-description': 'None', 'current-energy-efficiency': '32', + 'energy-consumption-current': '491', + 'co2-emissions-current': '5.0', + 'potential-energy-efficiency': '87' } ] From 0ede95cc4a7499ad0db1c6eda5ef6e012ab9f763 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:25:08 +0100 Subject: [PATCH 23/44] added override to wall insulation --- backend/Property.py | 4 +++- recommendations/WallRecommendations.py | 15 ++++++++++++++- recommendations/recommendation_utils.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 3fac3667..d000be28 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1,4 +1,5 @@ import os +import ast from itertools import groupby import pandas as pd @@ -77,7 +78,8 @@ class Property: # This is a list of measures that have already been installed in the property, typically found as a result # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the # cost and instead, provide a message that the measure has already been installed - self.override = override + + self.override = ast.literal_eval(override['overrides']) if override is not None else [] self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 6b59c148..3acc17f0 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -8,7 +8,7 @@ from backend.Property import Property from BaseUtility import Definitions from recommendations.recommendation_utils import ( r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, - get_recommended_part, get_wall_u_value + get_recommended_part, get_wall_u_value, override_costs ) from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION from recommendations.Costs import Costs @@ -221,6 +221,10 @@ class WallRecommendations(Definitions): material=material.to_dict(), ) + is_override = "cavity_wall_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) + recommendations.append( { "phase": phase, @@ -237,6 +241,7 @@ class WallRecommendations(Definitions): "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, + "is_override": is_override, **cost_result } ) @@ -277,12 +282,19 @@ class WallRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) + is_override = "internal_wall_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) + elif material["type"] == "external_wall_insulation": cost_result = self.costs.external_wall_insulation( wall_area=self.property.insulation_wall_area, material=material.to_dict(), non_insulation_materials=non_insulation_materials ) + is_override = "external_wall_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) else: raise ValueError("Invalid material type") @@ -301,6 +313,7 @@ class WallRecommendations(Definitions): "description": self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, + "is_override": is_override, "sap_points": None, **cost_result } diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 0d5f9743..a3043c31 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -767,3 +767,15 @@ def check_simulation_difference(old_config, new_config): differences = {key + "_ending": new_config[key] for key in new_config if old_config[key] != new_config[key]} return differences + + +def override_costs(costs): + """ + If the method is overridden, we want to make sure that the costs are zero. This function sets the costs to zero + :param costs: Dictionary of costing, as returned by the Costs class + :return: + """ + for k in costs: + costs[k] = 0 + + return costs From 1c5ccb2c8c46a613851dfaf153a16ee4242eaf0a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:26:11 +0100 Subject: [PATCH 24/44] added override to roof insulation --- recommendations/RoofRecommendations.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 8d6a91e7..ed087228 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -5,7 +5,7 @@ from typing import List from datatypes.enums import QuantityUnits from recommendations.recommendation_utils import ( get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, - update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric + update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric, override_costs ) from recommendations.Costs import Costs @@ -207,12 +207,18 @@ class RoofRecommendations: floor_area=self.property.insulation_floor_area, material=material ) + is_override = "loft_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) elif material["type"] == "flat_roof_insulation": cost_result = self.costs.flat_roof_insulation( floor_area=self.property.insulation_floor_area, material=material, non_insulation_materials=non_insulation_materials ) + is_override = "flat_roof_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) else: raise ValueError("Invalid material type") @@ -232,6 +238,7 @@ class RoofRecommendations: "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, + "is_override": is_override, **cost_result } ) From adcd31c8f4e69e92ff592a03103eb60f1c06617a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:27:58 +0100 Subject: [PATCH 25/44] correcting override in walls and roof --- recommendations/RoofRecommendations.py | 4 ++-- recommendations/VentilationRecommendations.py | 4 ++++ recommendations/WallRecommendations.py | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index ed087228..5ba7e82e 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -207,7 +207,7 @@ class RoofRecommendations: floor_area=self.property.insulation_floor_area, material=material ) - is_override = "loft_insulation" in cost_result + is_override = "loft_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) elif material["type"] == "flat_roof_insulation": @@ -216,7 +216,7 @@ class RoofRecommendations: material=material, non_insulation_materials=non_insulation_materials ) - is_override = "flat_roof_insulation" in cost_result + is_override = "flat_roof_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) else: diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 1657b759..aa6299e0 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -56,6 +56,10 @@ class VentilationRecommendations(Definitions): part[0]["quantity"] = n_units part[0]["quantity_unit"] = "part" + is_override = "cavity_wall_insulation" in cost_result + if is_override: + cost_result = override_costs(cost_result) + # We recommend installing two mechanical ventilation systems self.recommendation = [ { diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 3acc17f0..471a62cb 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -221,7 +221,7 @@ class WallRecommendations(Definitions): material=material.to_dict(), ) - is_override = "cavity_wall_insulation" in cost_result + is_override = "cavity_wall_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) @@ -282,7 +282,7 @@ class WallRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) - is_override = "internal_wall_insulation" in cost_result + is_override = "internal_wall_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) @@ -292,7 +292,7 @@ class WallRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) - is_override = "external_wall_insulation" in cost_result + is_override = "external_wall_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) else: From fadff714d2c3227eb835b94951ed09b25ff870c4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:29:41 +0100 Subject: [PATCH 26/44] add override to ventilation --- recommendations/VentilationRecommendations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index aa6299e0..07f7cf1e 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -50,16 +50,16 @@ class VentilationRecommendations(Definitions): part = self.materials.copy() - estimated_cost = n_units * part[0]["cost"] + is_override = "cavity_wall_insulation" in self.property.override + + estimated_cost = n_units * part[0]["cost"] if not is_override else 0 + labour_hours = 4 * n_units if not is_override else 0 + labour_days = 4 * n_units / 8.0 if not is_override else 0 part[0]["total"] = estimated_cost part[0]["quantity"] = n_units part[0]["quantity_unit"] = "part" - is_override = "cavity_wall_insulation" in cost_result - if is_override: - cost_result = override_costs(cost_result) - # We recommend installing two mechanical ventilation systems self.recommendation = [ { @@ -76,7 +76,7 @@ class VentilationRecommendations(Definitions): "energy_cost_savings": 0, "total": estimated_cost, # We use a very simple and rough estimate of 4 hours per unit - "labour_hours": 4 * n_units, - "labour_days": 4 * n_units / 8.0 # Assume 8 hour day + "labour_hours": labour_hours, + "labour_days": labour_days # Assume 8 hour day } ] From 493db6c4a01dcf825fe49d77cfc8fcb974a7d1e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:31:07 +0100 Subject: [PATCH 27/44] added floor insulation to override --- recommendations/FloorRecommendations.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 713d5f92..1744a928 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -8,7 +8,7 @@ from datatypes.enums import QuantityUnits from backend.Property import Property from recommendations.recommendation_utils import ( r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, - get_recommended_part, get_floor_u_value + get_recommended_part, get_floor_u_value, override_costs ) from recommendations.Costs import Costs @@ -192,12 +192,22 @@ class FloorRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) + + is_override = "suspended_floor_insulation" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + elif material["type"] == "solid_floor_insulation": cost_result = self.costs.solid_floor_insulation( insulation_floor_area=self.property.insulation_floor_area, material=material.to_dict(), non_insulation_materials=non_insulation_materials ) + + is_override = "solid_floor_insulation" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + else: raise NotImplementedError("Implement me!") From b052c9925f9064d2462442cccecac08bc511cc21 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:41:52 +0100 Subject: [PATCH 28/44] Added heating override --- recommendations/FireplaceRecommendations.py | 4 ++- recommendations/FloorRecommendations.py | 2 +- recommendations/HeatingRecommender.py | 22 ++++++++++++----- recommendations/WindowsRecommendations.py | 27 +++++++++++++-------- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 5d620d49..c1114f31 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -32,7 +32,8 @@ class FireplaceRecommendations(Definitions): if number_open_fireplaces == 0: return - estimated_cost = number_open_fireplaces * self.COST_OF_WORK + is_override = "sealing_open_fireplace" in self.property.override + estimated_cost = number_open_fireplaces * self.COST_OF_WORK if not is_override else 0 # We recommend installing two mechanical ventilation systems self.recommendation = [ @@ -44,6 +45,7 @@ class FireplaceRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, "total": estimated_cost, # Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal "labour_hours": 6 * number_open_fireplaces, diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 1744a928..b7bd370c 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -207,7 +207,6 @@ class FloorRecommendations(Definitions): is_override = "solid_floor_insulation" in self.property.override if is_override: cost_result = override_costs(cost_result) - else: raise NotImplementedError("Implement me!") @@ -227,6 +226,7 @@ class FloorRecommendations(Definitions): "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, + "is_override": is_override, **cost_result } ) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 92457a27..27e4985a 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,7 +1,7 @@ import pandas as pd from recommendations.Costs import Costs -from recommendations.recommendation_utils import check_simulation_difference +from recommendations.recommendation_utils import check_simulation_difference, override_costs from backend.Property import Property from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes @@ -33,7 +33,7 @@ class HeatingRecommender: if has_electric_heating_description or no_heating_no_mains: # Recommend high heat retention storage heaters - self.recommend_electric_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) + self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) # if the property has mains heating with boiler and radiators, we recommend optimal heating controls has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"] @@ -89,9 +89,8 @@ class HeatingRecommender: return differences - @staticmethod def combine_heating_and_controls( - controls_recommendations, heating_simulation_config, costs, description, phase, heating_controls_only, + self, controls_recommendations, heating_simulation_config, costs, description, phase, heating_controls_only, system_change ): """ @@ -140,6 +139,11 @@ class HeatingRecommender: recommendation_description = f"{description} and {controls_description}" + is_override = "cavity_wall_insulation" in self.property.override + if is_override: + total_costs = override_costs(total_costs) + recommendation_description = "Heating system has already been upgraded, no further action needed." + recommendation = { "phase": phase, "parts": [ @@ -150,6 +154,7 @@ class HeatingRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, **total_costs, "simulation_config": recommendation_simulation_config } @@ -181,9 +186,8 @@ class HeatingRecommender: return output - def recommend_electric_storage_heaters(self, phase, system_change, heating_controls_only): + def recommend_hhr_storage_heaters(self, phase, system_change, heating_controls_only): """ - We recommend electric storage heaters as an upgrade to the heating system. We will recommend upgrading to a high heat retention storage system, if the current system is not already high heat retention storage @@ -360,6 +364,11 @@ class HeatingRecommender: n_heated_rooms=self.property.data["number-heated-rooms"] ) + is_override = "heating" in self.property.override + if is_override: + boiler_costs = override_costs(boiler_costs) + description = "Heating system has already been upgraded, no further action needed." + boiler_recommendation = { "phase": recommendation_phase, "parts": [ @@ -370,6 +379,7 @@ class HeatingRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, "simulation_config": simulation_config, **boiler_costs } diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index d7404e3b..b2fe20a6 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -4,6 +4,7 @@ import numpy as np from backend.Property import Property from recommendations.Costs import Costs +from recommendation_utils import override_costs class WindowsRecommendations: @@ -70,18 +71,23 @@ class WindowsRecommendations: is_secondary_glazing=is_secondary_glazing ) - glazing_type = "secondary glazing" if is_secondary_glazing else "double glazing" - if self.property.windows["glazing_coverage"] in ["partial", "most"]: - description = f"Install {glazing_type} to the remaining windows" + is_override = "windows_glazing" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + description = "The property already has double glazing installed. No further action is required." else: - description = f"Install {glazing_type} to all windows" + glazing_type = "secondary glazing" if is_secondary_glazing else "double glazing" + if self.property.windows["glazing_coverage"] in ["partial", "most"]: + description = f"Install {glazing_type} to the remaining windows" + else: + description = f"Install {glazing_type} to all windows" - if self.property.is_listed: - description += ". Secondary glazing recommended due to listed building status" - elif self.property.is_heritage: - description += ". Secondary glazing recommended due to herigate building status" - elif self.property.in_conservation_area: - description += ". Secondary glazing recommended due to conservation area status" + if self.property.is_listed: + description += ". Secondary glazing recommended due to listed building status" + elif self.property.is_heritage: + description += ". Secondary glazing recommended due to herigate building status" + elif self.property.in_conservation_area: + description += ". Secondary glazing recommended due to conservation area status" self.recommendation = [ { @@ -92,6 +98,7 @@ class WindowsRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, **cost_result, "is_secondary_glazing": is_secondary_glazing } From 1ee115fa7e73f170d559a24026680677f89aaf5d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 15:48:44 +0100 Subject: [PATCH 29/44] Added overrides --- recommendations/HotwaterRecommendations.py | 11 ++++++++++- recommendations/LightingRecommendations.py | 7 +++++++ recommendations/SecondaryHeating.py | 12 +++++++++++- recommendations/SolarPvRecommendations.py | 6 ++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/recommendations/HotwaterRecommendations.py b/recommendations/HotwaterRecommendations.py index 7f77597f..88cfa932 100644 --- a/recommendations/HotwaterRecommendations.py +++ b/recommendations/HotwaterRecommendations.py @@ -1,5 +1,6 @@ from backend.Property import Property from recommendations.Costs import Costs +from recommendations.recommendation_utils import override_costs class HotwaterRecommendations: @@ -41,6 +42,13 @@ class HotwaterRecommendations: recommendation_cost = self.costs.hot_water_tank_insulation() + is_override = "hot_water_tank_insulation" in self.property.override + if is_override: + recommendation_cost = override_costs(recommendation_cost) + description = "Insulation tank has already been insulated, no further action required" + else: + description = "Insulate hot water tank" + self.recommendations.append( { "phase": phase, @@ -48,10 +56,11 @@ class HotwaterRecommendations: # TODO ], "type": "hot_water_tank_insulation", - "description": "Insulate the hot water tank with an insulation jacket", + "description": description, "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, **recommendation_cost, "simulation_config": {"hot_water_energy_eff_ending": "Average"} } diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 352c4d8a..9e4c8e43 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -1,6 +1,7 @@ from backend.Property import Property from typing import List from recommendations.Costs import Costs +from recommendations.recommendation_utils import override_costs class LightingRecommendations: @@ -91,6 +92,11 @@ class LightingRecommendations: heat_demand_change, carbon_change = self.estimate_lighting_impact(number_non_lel_outlets) + is_override = "low_energy_lighting" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + description = "Low energy lighting has already been installed, no further action required" + self.recommendation = [ { "phase": phase, @@ -99,6 +105,7 @@ class LightingRecommendations: "description": description, "starting_u_value": None, "new_u_value": None, + "is_override": is_override, # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to # the proportion of lights that will be set to low energy "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2), diff --git a/recommendations/SecondaryHeating.py b/recommendations/SecondaryHeating.py index f31c4c05..e426977e 100644 --- a/recommendations/SecondaryHeating.py +++ b/recommendations/SecondaryHeating.py @@ -1,4 +1,5 @@ from recommendations.Costs import Costs +from recommendations.recommendation_utils import override_costs from backend.Property import Property @@ -38,15 +39,24 @@ class SecondaryHeating: n_rooms = 0 costs = self.costs.heater_removal(n_rooms=n_rooms) + + is_override = "secondary_heating" in self.property.override + if is_override: + costs = override_costs(costs) + description = "Secondary heating system has already been removed, no further action required" + else: + description = "Remove the secondary heating system" + self.recommendation.append( { "phase": phase, "parts": [], "type": "secondary_heating", - "description": "Remove the secondary heating system", + "description": description, "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, **costs, "simulation_config": { "secondheat_description_ending": "None" diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index f75003ce..72fcdf4b 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -1,5 +1,6 @@ import numpy as np from recommendations.Costs import Costs +from recommendations.recommendation_utils import override_costs class SolarPvRecommendations: @@ -110,6 +111,10 @@ class SolarPvRecommendations: description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p" f"anel system on {round(roof_coverage_percent)}% the roof.") + is_override = "solar_pv" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + self.recommendation.append( { "phase": phase, @@ -119,6 +124,7 @@ class SolarPvRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, **cost_result, # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale # back up here From 14a1f35fb16cbf1199afbd66ce50f598b5d7a10b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 16:27:26 +0100 Subject: [PATCH 30/44] ammended system change costs for first time central heating --- recommendations/Costs.py | 72 +++++++++++++++++++++++++-- recommendations/HeatingRecommender.py | 9 +++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 45c17102..0e67b352 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -83,6 +83,14 @@ CONVENTIONAL_BOILER_COSTS = { ROOM_HEATER_REMOVAL_COST = 120 ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3 +# This is a cost quoted by Jim for a system flush - existig system will run more efficiently +SYSTEM_FLUSH_COST = 250 + +SINGLE_RADIATOR_COST = 150 +DOUBLE_RADIATOR_COST = 300 +FLUE_COST = 600 +PIPEWORK_COST = 750 # Min cost is £500 + class Costs: """ @@ -1126,9 +1134,45 @@ class Costs: "labour_days": np.ceil(removal_labour_hours / 8), } - def boiler(self, is_combi, size, exising_room_heaters, n_heated_rooms): + @staticmethod + def _estimate_n_radiators(number_habitable_rooms, total_floor_area, property_type, built_form): + # Base number of radiators: one per habitable room + base_radiators = number_habitable_rooms + + # Additional radiators for non-habitable essential areas (e.g., kitchens, hallways) + additional_radiators = 3 # Initial assumption + + # Adjust additional radiators based on property type + if property_type == 'Flat': + additional_radiators -= 1 # Flats may need fewer radiators due to less exposure + elif property_type in ['House', 'Bungalow', 'Maisonette']: + # Multiple floors in Maisonette may require additional heating points + additional_radiators += 2 # Houses and bungalows might need more due to greater exposure + else: + raise Exception("Invalid property type") + + # Adjust total radiator needs based on built form + form_factor = { + 'Mid-Terrace': 0.95, + 'Semi-Detached': 1.05, + 'Detached': 1.25, + 'End-Terrace': 1.05 + } + + # Calculate total heating power needed and number of radiators based on standard output + total_heating_power_required = total_floor_area * 80 # Watts per square meter + radiator_output = 1000 # Average wattage per radiator + total_radiators_based_on_power = (total_heating_power_required / radiator_output) * form_factor[built_form] + + # Final estimation taking the higher of calculated needs or base room count + estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators) + return round(estimated_radiators) + + def boiler(self, is_combi, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms): """ Based on a basic estimate of median value £2600 to install a low carbon combi boiler + First time central heating vosts can als be found here: + https://www.checkatrade.com/blog/cost-guides/central-heating-installation-cost/ :return: """ @@ -1137,11 +1181,11 @@ class Costs: # We now need to estimate the cost of the works labour_days = 2 labour_hours = labour_days * 8 - labour_rate = 500 + labour_rate = 300 # Average cost of installation is 1 (maybe 2days) at £300 per day # https://www.checkatrade.com/blog/cost-guides/new-boiler-cost/ - # To be pessimistic, assume 2 days work and £500 day rate + # To be pessimistic, assume 2 days work labour_cost = labour_rate * self.labour_adjustment_factor * labour_days # Add contingency and preliminaries labour_cost = labour_cost * (1 + self.CONTINGENCY + self.PRELIMINARIES) @@ -1161,6 +1205,28 @@ class Costs: subtotal_before_vat += removal_costing["subtotal"] labour_hours += removal_costing["labour_hours"] labour_days += removal_costing["labour_days"] + vat += removal_costing["vat"] + + if system_change: + # We need the cost of radiators + n_radiators = self._estimate_n_radiators( + number_habitable_rooms=n_rooms, + total_floor_area=self.property.floor_area, + property_type=self.property.data["property-type"], + built_form=self.property.data["built-form"] + ) + + additionals_labour_cost = labour_rate * self.labour_adjustment_factor + radiator_cost = DOUBLE_RADIATOR_COST * n_radiators + system_change_cost = radiator_cost + FLUE_COST + PIPEWORK_COST + additionals_labour_cost + system_change_cost_before_vat = system_change_cost / (1 + self.VAT_RATE) + system_change_vat = system_change_cost - system_change_cost_before_vat + # We add an extra labour day for the system change + labour_days += 1 + labour_hours += 8 + total_cost += system_change_cost + subtotal_before_vat += system_change_cost_before_vat + vat += system_change_vat return { "total": total_cost, diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 27e4985a..d83b755e 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -18,6 +18,11 @@ class HeatingRecommender: self.recommendations = [] def recommend(self, phase=0): + + # TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace + # the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this + # in the Costs class, stored as SYSTEM_FLUSH_COST + self.recommendations = [] # This first iteration of the recommender will provide very basic recommendation # We recommend heating controls based on the main heating system @@ -361,7 +366,9 @@ class HeatingRecommender: is_combi=is_combi, size=f"{boiler_size}kw", exising_room_heaters=exising_room_heaters, - n_heated_rooms=self.property.data["number-heated-rooms"] + system_change=system_change, + n_heated_rooms=self.property.data["number-heated-rooms"], + n_rooms=self.property.number_of_rooms ) is_override = "heating" in self.property.override From 94f9979f561c5a64acea1fc871c38a9d4868f8e0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 16:31:11 +0100 Subject: [PATCH 31/44] fixed override bug --- backend/Property.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Property.py b/backend/Property.py index d000be28..2892b86e 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -79,7 +79,7 @@ class Property: # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the # cost and instead, provide a message that the measure has already been installed - self.override = ast.literal_eval(override['overrides']) if override is not None else [] + self.override = ast.literal_eval(override['overrides']) if override else [] self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") From d8caacae97006638aed112e7c8682a0a23372690 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 12 Apr 2024 17:46:06 +0100 Subject: [PATCH 32/44] creating non-invasive survey results WIP --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- etl/customers/immo/pilot/non_invasive.py | 131 ++++++++++++++++++++++ etl/customers/immo/pilot/requirements.txt | 1 + 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 etl/customers/immo/pilot/non_invasive.py create mode 100644 etl/customers/immo/pilot/requirements.txt diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py new file mode 100644 index 00000000..cb978059 --- /dev/null +++ b/etl/customers/immo/pilot/non_invasive.py @@ -0,0 +1,131 @@ +import extract_msg + + +def parse_msg_body(text): + # Split the text into lines + lines = text.split('\r\n') + + # Dictionary to hold the parsed data + data = {} + + # Process each line + for line in lines: + # Remove all asterisks and extra whitespace + clean_line = line.replace('*', '').strip() + + if clean_line: # Ensure the line is not empty after cleaning + # Attempt to split clean '=' if present + if '=' in clean_line: + clean_line = clean_line.replace(' = ', ': ') + + # Use line content as a key with a default value indicating presence + # Generate a unique key for lines without '=' + data[f"Info{len(data) + 1}"] = clean_line + + return data + + +def app(): + """ + This code retrieves the results of the non-invasive surveys, to be stored in S3 + :return: + """ + + # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/5 Oaklands B62 " + # "0JA/Immo - 5 Oaklands Halesowen B62 0JA.msg") + # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/6 Beech Rd DY1 " + # "4BP/IMMO - 6 Beech Road Dudley DY1 4BP.msg") + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/8 Corporation Rd DY2 " + # "7PX/IMMO - 8 Corporation Road Dudley DY2 7PX.msg" + # ) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/21 Wells Rd DY5 3TB/" + # "IMMO - 21 Wells Road Brierley Hill DY5 3TB.msg" + # ) + filepath = ( + "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/47 Fairfield Rd DY8 " + "5UJ/IMMO - 47 Fairfield Road Wordsley Stourbridge DY8 5UJ.msg" + ) + + with extract_msg.Message(filepath) as msg: + sender = msg.sender + recipients = msg.to + subject = msg.subject + body = msg.body + # If the msg has attachments, they can be extracted as well + attachments = msg.attachments + + from pprint import pprint + pprint(parse_msg_body(body)) + + # We manually create the non-invasive notes for the pilot + non_invasive_notes = [ + { + 'address': '5 Oaklands', + 'postcode': 'B62 0JA', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation. ' + 'There is a shared alleyway with the neighbour, that is a solid brick wall.', + 'Wall Render': 'Partial render between top of ground floor window and bottom of 1st floor window', + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: North East, Back house direction: South West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '6 Beech Road', + 'postcode': 'DY1 4BP', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, ' + 'retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Side house direction: North East', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '8 Corporation Road', + 'postcode': 'DY2 7PX', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': "External wall insulation", + 'Wall Render': "Render finish throughout", + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: North East, Back house direction: South West', + 'Access to mains?': None, + }, + { + + 'address': '21 Wells Road', + 'postcode': 'DY5 3TB', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: East, Back house direction: West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '47 Fairfield Road', + 'postcode': 'DY8 5UJ', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': None, + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: East, Back house direction: West', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': None, + 'postcode': None, + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': None, + 'Wall Render': None, + 'Existing solar PV': None, + 'Orientation': None, + 'Access to mains?': None, + }, + ] diff --git a/etl/customers/immo/pilot/requirements.txt b/etl/customers/immo/pilot/requirements.txt new file mode 100644 index 00000000..4673ab35 --- /dev/null +++ b/etl/customers/immo/pilot/requirements.txt @@ -0,0 +1 @@ +extract-msg From a158f2353c0f84bb005924441166ef56a899f59c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 15:36:58 +0100 Subject: [PATCH 33/44] manually created non-invasie notes --- etl/customers/immo/pilot/non_invasive.py | 63 ++++++++++++++++++++---- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py index cb978059..c2b8ea64 100644 --- a/etl/customers/immo/pilot/non_invasive.py +++ b/etl/customers/immo/pilot/non_invasive.py @@ -43,9 +43,17 @@ def app(): # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/21 Wells Rd DY5 3TB/" # "IMMO - 21 Wells Road Brierley Hill DY5 3TB.msg" # ) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/47 Fairfield Rd DY8 " + # "5UJ/IMMO - 47 Fairfield Road Wordsley Stourbridge DY8 5UJ.msg" + # ) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/91 Osprey Drive DY1 " + # "2JS/IMMO - 91 Osprey Drive Dudley DY1 2JS.msg" + # ) filepath = ( - "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/47 Fairfield Rd DY8 " - "5UJ/IMMO - 47 Fairfield Road Wordsley Stourbridge DY8 5UJ.msg" + "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " + "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" ) with extract_msg.Message(filepath) as msg: @@ -119,13 +127,50 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { - 'address': None, - 'postcode': None, + 'address': '53 Bromley', + 'postcode': 'DY5 4PJ', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', - 'Wall Insulation': None, - 'Wall Render': None, - 'Existing solar PV': None, - 'Orientation': None, - 'Access to mains?': None, + 'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - " + "recommends a cavity wall fill", + "Roof": "Hipped roof", + 'Existing solar PV': 'No existing solar', + 'Orientation': "Front house direction: North, Back house direction: South, Side house direction: West", + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '91 Osprey Drive', + 'postcode': 'DY1 2JS', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': 'Tile hung front and rear of property', + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Side house direction: East', + 'Access to mains?': 'Property has access to the mains', + }, + { + 'address': '150 Huntingtree Road', + 'postcode': 'B63 4HP', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Heating': 'Electric (storage heaters)', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + "Roof": "Hipped roof", + 'Existing solar PV': 'No existing solar', + 'Orientation': "Front house direction: North West, Back house direction: South East, Side house direction: " + "North East", + }, + { + 'address': '195 Ashenhurst Road', + 'postcode': 'DY1 2JB', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': "Solid render front and rear of property", + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: South, Back house direction: North', + 'Access to mains?': 'Property has access to the mains', }, ] + + # TODO: Push the non-invasive results straight to the database from here From 485c01cbd69cf8b562b2d53da0ae03915edf8d93 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 16:14:01 +0100 Subject: [PATCH 34/44] Added uprns to non-invaive notes --- etl/customers/immo/pilot/non_invasive.py | 35 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py index c2b8ea64..0a376388 100644 --- a/etl/customers/immo/pilot/non_invasive.py +++ b/etl/customers/immo/pilot/non_invasive.py @@ -51,18 +51,17 @@ def app(): # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/91 Osprey Drive DY1 " # "2JS/IMMO - 91 Osprey Drive Dudley DY1 2JS.msg" # ) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " + # "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" + # ) filepath = ( - "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " - "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" + "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " + "- 27 Milton Road Coseley Bilston WV14 8HZ.msg" ) with extract_msg.Message(filepath) as msg: - sender = msg.sender - recipients = msg.to - subject = msg.subject body = msg.body - # If the msg has attachments, they can be extracted as well - attachments = msg.attachments from pprint import pprint pprint(parse_msg_body(body)) @@ -70,6 +69,7 @@ def app(): # We manually create the non-invasive notes for the pilot non_invasive_notes = [ { + 'uprn': 90028499, 'address': '5 Oaklands', 'postcode': 'B62 0JA', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -82,6 +82,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90055152, 'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -94,6 +95,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90070461, 'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -104,7 +106,7 @@ def app(): 'Access to mains?': None, }, { - + 'uprn': 90022227, 'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -116,6 +118,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90077535, 'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -127,6 +130,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90060989, 'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -138,6 +142,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90048026, 'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -149,6 +154,7 @@ def app(): 'Access to mains?': 'Property has access to the mains', }, { + 'uprn': 90093693, 'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -161,6 +167,7 @@ def app(): "North East", }, { + 'uprn': 90051858, 'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', @@ -171,6 +178,18 @@ def app(): 'Orientation': 'Front house direction: South, Back house direction: North', 'Access to mains?': 'Property has access to the mains', }, + { + 'uprn': 90106884, + 'address': '27 Milton Road', + 'postcode': 'WV14 8HZ', + 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' + 'CIGA check and extracting the cavity, replacing with bead insulation.', + 'Wall Render': "Solid render front and rear of property", + 'Existing solar PV': 'No existing solar', + 'Orientation': 'Front house direction: South East, Back house direction: North West', + 'Access to mains?': 'Property has access to the mains', + }, ] # TODO: Push the non-invasive results straight to the database from here From 65f83930d56290fc73846ca4c8626ac46e3cd7c6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 16:25:12 +0100 Subject: [PATCH 35/44] added is_override to storage of recommendation --- .../db/functions/recommendations_functions.py | 3 ++- .../app/db/models/non_intrusive_surveys.py | 24 +++++++++++++++++++ backend/app/db/models/recommendations.py | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 backend/app/db/models/non_intrusive_surveys.py diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 1426e339..43daec77 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -85,7 +85,8 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "co2_equivalent_savings": rec["co2_equivalent_savings"], "total_work_hours": rec["labour_hours"], "energy_cost_savings": rec["energy_cost_savings"], - "labour_days": rec["labour_days"] + "labour_days": rec["labour_days"], + "is_override": rec["is_override"], } for rec in recommendations_to_upload ] diff --git a/backend/app/db/models/non_intrusive_surveys.py b/backend/app/db/models/non_intrusive_surveys.py new file mode 100644 index 00000000..c5f3734a --- /dev/null +++ b/backend/app/db/models/non_intrusive_surveys.py @@ -0,0 +1,24 @@ +from sqlalchemy import Column, BigInteger, String, Float, Boolean, TIMESTAMP, ForeignKey, Enum, Integer +from sqlalchemy.orm import declarative_base +from sqlalchemy.sql import func +from backend.app.db.models.portfolio import Portfolio, PropertyModel +from backend.app.db.models.materials import Material +from datatypes.enums import QuantityUnits + +Base = declarative_base() + + +class NonIntrusiveSurvey(Base): + __tablename__ = 'non_intrusive_survey' + + id = Column(BigInteger, primary_key=True, autoincrement=True) + uprn = Column(Integer, nullable=False) + survey_date = Column(TIMESTAMP, nullable=False) + surveyor = Column(String, nullable=False) + + +class NonIntrusiveSurveyNotes(Base): + id = Column(BigInteger, primary_key=True, autoincrement=True) + survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False) + title = Column(String, nullable=False) + note = Column(String, nullable=False) diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index a492f2f2..be5ff30c 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -30,6 +30,7 @@ class Recommendation(Base): rental_yield_increase = Column(Float) total_work_hours = Column(Float) labour_days = Column(Float) + is_override = Column(Boolean, nullable=False, default=False) class RecommendationMaterials(Base): From aaa279463eea2505b3d36ee46c26b33b17955e77 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 16:37:28 +0100 Subject: [PATCH 36/44] Added is_override to heating controls --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- recommendations/HeatingControlRecommender.py | 46 ++++++++++++++------ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 76eaba4f..63218163 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -1,5 +1,5 @@ from recommendations.Costs import Costs -from recommendations.recommendation_utils import check_simulation_difference +from recommendations.recommendation_utils import check_simulation_difference, override_costs from backend.Property import Property from etl.epc_clean.epc_attributes.MainheatControlAttributes import MainheatControlAttributes @@ -159,20 +159,30 @@ class HeatingControlRecommender: has_room_thermostat = not needs_room_thermostat has_trvs = not needs_trvs + cost_result = self.costs.roomstat_programmer_trvs( + number_heated_rooms=int(self.property.data["number-heated-rooms"]), + has_programmer=has_programmer, + has_room_thermostat=has_room_thermostat, + has_trvs=has_trvs + ) + + description = "upgrade heating controls to Room thermostat, programmer and TRVs" + + is_override = "heating_control" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + description = "Heating controls have already been upgraded, no further action needed." + self.recommendation.append( { "type": "heating_control", "parts": [], - "description": "upgrade heating controls to Room thermostat, programmer and TRVs", - **self.costs.roomstat_programmer_trvs( - number_heated_rooms=int(self.property.data["number-heated-rooms"]), - has_programmer=has_programmer, - has_room_thermostat=has_room_thermostat, - has_trvs=has_trvs - ), + "description": description, + **cost_result, "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, "simulation_config": simulation_config } ) @@ -211,18 +221,28 @@ class HeatingControlRecommender: if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average", "Good"]: simulation_config["mainheatc_energy_eff_ending"] = "Very Good" + cost_result = self.costs.time_and_temperature_zone_control( + number_heated_rooms=int(self.property.data["number-heated-rooms"]) + ) + + description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " + "temperature zone control)") + + is_override = "heating_control" in self.property.override + if is_override: + cost_result = override_costs(cost_result) + description = "Heating controls have already been upgraded, no further action needed." + self.recommendation.append( { "type": "heating_control", "parts": [], - "description": "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves " - "(time & temperature zone control)", - **self.costs.time_and_temperature_zone_control( - number_heated_rooms=int(self.property.data["number-heated-rooms"]) - ), + "description": description, + **cost_result, "starting_u_value": None, "new_u_value": None, "sap_points": None, + "is_override": is_override, "simulation_config": simulation_config } ) From 527291b4395eb8b5563f52fd8449faee569d6789 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 16:40:13 +0100 Subject: [PATCH 37/44] Added is_override to mechanical ventilation recommendation --- recommendations/VentilationRecommendations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 07f7cf1e..7ffcda08 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -69,6 +69,7 @@ class VentilationRecommendations(Definitions): "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, + "is_override": is_override, "sap_points": 0, "heat_demand": 0, "adjusted_heat_demand": 0, From 34d6a075289b0c2d31d75a1bad8ea5c969f12fca Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 13 Apr 2024 17:07:42 +0100 Subject: [PATCH 38/44] Pushed non-intrusive survey results to bd --- .../app/db/functions/non_intrusive_surveys.py | 50 ++++++++++ .../app/db/models/non_intrusive_surveys.py | 8 +- etl/customers/immo/pilot/non_invasive.py | 99 +++++++++++-------- 3 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 backend/app/db/functions/non_intrusive_surveys.py diff --git a/backend/app/db/functions/non_intrusive_surveys.py b/backend/app/db/functions/non_intrusive_surveys.py new file mode 100644 index 00000000..93348121 --- /dev/null +++ b/backend/app/db/functions/non_intrusive_surveys.py @@ -0,0 +1,50 @@ +from sqlalchemy.orm import Session +from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes + + +def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500): + """ + Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents + one survey and its associated notes. + + :param session: SQLAlchemy Session object through which all database transactions are handled. + :param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn', + 'survey_date', 'surveyor', and other notes as key-value pairs. + :param batch_size: The size of each batch to be processed (default is 500). + :return: None + """ + + # Helper function to process each batch + def process_batch(batch): + surveys = [] + notes = [] + + for note in batch: + survey = NonIntrusiveSurvey( + uprn=note['uprn'], + survey_date=note['survey_date'], + surveyor=note['surveyor'] + ) + surveys.append(survey) + + session.add_all(surveys) + session.flush() # Get IDs for surveys + + for note, survey in zip(batch, surveys): + for key, value in note.items(): + if key not in ['uprn', 'survey_date', 'surveyor']: + notes.append(NonIntrusiveSurveyNotes( + survey_id=survey.id, + title=key, + note=value + )) + + session.bulk_save_objects(notes) + session.commit() + + # Split the data into batches and process each batch + total = len(non_invasive_notes) + for start in range(0, total, batch_size): + end = min(start + batch_size, total) + batch = non_invasive_notes[start:end] + process_batch(batch) diff --git a/backend/app/db/models/non_intrusive_surveys.py b/backend/app/db/models/non_intrusive_surveys.py index c5f3734a..bc2d8adc 100644 --- a/backend/app/db/models/non_intrusive_surveys.py +++ b/backend/app/db/models/non_intrusive_surveys.py @@ -1,9 +1,5 @@ -from sqlalchemy import Column, BigInteger, String, Float, Boolean, TIMESTAMP, ForeignKey, Enum, Integer +from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer from sqlalchemy.orm import declarative_base -from sqlalchemy.sql import func -from backend.app.db.models.portfolio import Portfolio, PropertyModel -from backend.app.db.models.materials import Material -from datatypes.enums import QuantityUnits Base = declarative_base() @@ -18,6 +14,8 @@ class NonIntrusiveSurvey(Base): class NonIntrusiveSurveyNotes(Base): + __tablename__ = 'non_intrusive_survey_notes' + id = Column(BigInteger, primary_key=True, autoincrement=True) survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False) title = Column(String, nullable=False) diff --git a/etl/customers/immo/pilot/non_invasive.py b/etl/customers/immo/pilot/non_invasive.py index 0a376388..6dc22c62 100644 --- a/etl/customers/immo/pilot/non_invasive.py +++ b/etl/customers/immo/pilot/non_invasive.py @@ -1,4 +1,8 @@ -import extract_msg +# import extract_msg +from datetime import datetime +from sqlalchemy.orm import sessionmaker +from backend.app.db.connection import db_engine +from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes def parse_msg_body(text): @@ -55,24 +59,25 @@ def app(): # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 " # "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg" # ) - filepath = ( - "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " - "- 27 Milton Road Coseley Bilston WV14 8HZ.msg" - ) - - with extract_msg.Message(filepath) as msg: - body = msg.body - - from pprint import pprint - pprint(parse_msg_body(body)) + # filepath = ( + # "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO " + # "- 27 Milton Road Coseley Bilston WV14 8HZ.msg" + # ) + # + # with extract_msg.Message(filepath) as msg: + # body = msg.body + # + # from pprint import pprint + # pprint(parse_msg_body(body)) # We manually create the non-invasive notes for the pilot non_invasive_notes = [ { 'uprn': 90028499, - 'address': '5 Oaklands', - 'postcode': 'B62 0JA', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '5 Oaklands', + # 'postcode': 'B62 0JA', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation. ' 'There is a shared alleyway with the neighbour, that is a solid brick wall.', @@ -83,9 +88,10 @@ def app(): }, { 'uprn': 90055152, - 'address': '6 Beech Road', - 'postcode': 'DY1 4BP', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '6 Beech Road', + # 'postcode': 'DY1 4BP', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, ' 'retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', @@ -96,9 +102,10 @@ def app(): }, { 'uprn': 90070461, - 'address': '8 Corporation Road', - 'postcode': 'DY2 7PX', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '8 Corporation Road', + # 'postcode': 'DY2 7PX', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': "External wall insulation", 'Wall Render': "Render finish throughout", 'Existing solar PV': 'No existing solar', @@ -107,9 +114,10 @@ def app(): }, { 'uprn': 90022227, - 'address': '21 Wells Road', - 'postcode': 'DY5 3TB', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '21 Wells Road', + # 'postcode': 'DY5 3TB', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': None, @@ -119,9 +127,10 @@ def app(): }, { 'uprn': 90077535, - 'address': '47 Fairfield Road', - 'postcode': 'DY8 5UJ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '47 Fairfield Road', + # 'postcode': 'DY8 5UJ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': None, @@ -131,9 +140,10 @@ def app(): }, { 'uprn': 90060989, - 'address': '53 Bromley', - 'postcode': 'DY5 4PJ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '53 Bromley', + # 'postcode': 'DY5 4PJ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - " "recommends a cavity wall fill", "Roof": "Hipped roof", @@ -143,9 +153,10 @@ def app(): }, { 'uprn': 90048026, - 'address': '91 Osprey Drive', - 'postcode': 'DY1 2JS', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '91 Osprey Drive', + # 'postcode': 'DY1 2JS', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': 'Tile hung front and rear of property', @@ -155,9 +166,10 @@ def app(): }, { 'uprn': 90093693, - 'address': '150 Huntingtree Road', - 'postcode': 'B63 4HP', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '150 Huntingtree Road', + # 'postcode': 'B63 4HP', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Heating': 'Electric (storage heaters)', 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', @@ -168,9 +180,10 @@ def app(): }, { 'uprn': 90051858, - 'address': '195 Ashenhurst Road', - 'postcode': 'DY1 2JB', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '195 Ashenhurst Road', + # 'postcode': 'DY1 2JB', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': "Solid render front and rear of property", @@ -180,9 +193,10 @@ def app(): }, { 'uprn': 90106884, - 'address': '27 Milton Road', - 'postcode': 'WV14 8HZ', - 'Surveyor': 'Carl Fitzgerald - The Warmfront Team', + # 'address': '27 Milton Road', + # 'postcode': 'WV14 8HZ', + 'surveyor': 'Carl Fitzgerald - The Warmfront Team', + 'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'), 'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a ' 'CIGA check and extracting the cavity, replacing with bead insulation.', 'Wall Render': "Solid render front and rear of property", @@ -192,4 +206,5 @@ def app(): }, ] - # TODO: Push the non-invasive results straight to the database from here + session = sessionmaker(bind=db_engine)() + upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500) From 954fa9d32c5d30bd63098b74512b006b47bf3056 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 14 Apr 2024 14:57:13 +0100 Subject: [PATCH 39/44] changed is_override to already_installed in recommendations --- backend/Property.py | 4 ++-- backend/app/plan/router.py | 17 +++++++++-------- backend/app/plan/schemas.py | 2 +- recommendations/FireplaceRecommendations.py | 6 +++--- recommendations/FloorRecommendations.py | 10 +++++----- recommendations/HeatingControlRecommender.py | 12 ++++++------ recommendations/HeatingRecommender.py | 12 ++++++------ recommendations/HotwaterRecommendations.py | 6 +++--- recommendations/LightingRecommendations.py | 6 +++--- recommendations/RoofRecommendations.py | 10 +++++----- recommendations/SecondaryHeating.py | 6 +++--- recommendations/SolarPvRecommendations.py | 6 +++--- recommendations/VentilationRecommendations.py | 10 +++++----- recommendations/WallRecommendations.py | 16 ++++++++-------- recommendations/WindowsRecommendations.py | 6 +++--- 15 files changed, 65 insertions(+), 64 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 2892b86e..a8ed9129 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -61,7 +61,7 @@ class Property: n_bedrooms = None def __init__( - self, id, postcode, address, epc_record, override=None, **kwargs + self, id, postcode, address, epc_record, already_installed=None, **kwargs ): self.epc_record = epc_record @@ -79,7 +79,7 @@ class Property: # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the # cost and instead, provide a message that the measure has already been installed - self.override = ast.literal_eval(override['overrides']) if override else [] + self.already_installed = ast.literal_eval(already_installed['already_installed']) if already_installed else [] self.uprn = epc_record.get("uprn") self.full_sap_epc = epc_record.get("full_sap_epc") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 08ce0dcc..49e14872 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -74,7 +74,7 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: We should store the trigger file path in the database with the plan so we can track the file that # triggered the plan - # TODO: Create the ability to congigure/switch off certain measures + # TODO: if the measure is already installed, it should actually be the very first phase try: session.begin() @@ -85,10 +85,10 @@ async def trigger_plan(body: PlanTriggerRequest): if body.patches_file_path: patches = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.patches_file_path) - overrides = [] - if body.overrides_file_path: - overrides = read_csv_from_s3( - bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.overrides_file_path + already_installed = [] + if body.already_installed_file_path: + already_installed = read_csv_from_s3( + bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.already_installed_file_path ) cleaning_data = read_dataframe_from_s3_parquet( @@ -142,8 +142,9 @@ async def trigger_plan(body: PlanTriggerRequest): cleaning_data=cleaning_data ) - override = next(( - x for x in overrides if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) + property_already_installed = next(( + x for x in already_installed if + (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) ), {}) input_properties.append( Property( @@ -151,7 +152,7 @@ async def trigger_plan(body: PlanTriggerRequest): address=epc_searcher.address_clean, postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, - override=override, + already_installed=property_already_installed, **Property.extract_kwargs(config) ) ) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index ec49e41e..76eb49d2 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -9,7 +9,7 @@ class PlanTriggerRequest(BaseModel): goal_value: str portfolio_id: int trigger_file_path: str - overrides_file_path: Optional[str] = None + already_installed_file_path: Optional[str] = None patches_file_path: Optional[str] = None exclusions: Optional[conlist(str, min_items=1)] = None diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index c1114f31..601a8eb0 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -32,8 +32,8 @@ class FireplaceRecommendations(Definitions): if number_open_fireplaces == 0: return - is_override = "sealing_open_fireplace" in self.property.override - estimated_cost = number_open_fireplaces * self.COST_OF_WORK if not is_override else 0 + already_installed = "sealing_open_fireplace" in self.property.already_installed + estimated_cost = number_open_fireplaces * self.COST_OF_WORK if not already_installed else 0 # We recommend installing two mechanical ventilation systems self.recommendation = [ @@ -45,7 +45,7 @@ class FireplaceRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, "total": estimated_cost, # Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal "labour_hours": 6 * number_open_fireplaces, diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index b7bd370c..3f764d83 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -193,8 +193,8 @@ class FloorRecommendations(Definitions): non_insulation_materials=non_insulation_materials ) - is_override = "suspended_floor_insulation" in self.property.override - if is_override: + already_installed = "suspended_floor_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) elif material["type"] == "solid_floor_insulation": @@ -204,8 +204,8 @@ class FloorRecommendations(Definitions): non_insulation_materials=non_insulation_materials ) - is_override = "solid_floor_insulation" in self.property.override - if is_override: + already_installed = "solid_floor_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) else: raise NotImplementedError("Implement me!") @@ -226,7 +226,7 @@ class FloorRecommendations(Definitions): "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **cost_result } ) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 63218163..d24ad811 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -168,8 +168,8 @@ class HeatingControlRecommender: description = "upgrade heating controls to Room thermostat, programmer and TRVs" - is_override = "heating_control" in self.property.override - if is_override: + already_installed = "heating_control" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) description = "Heating controls have already been upgraded, no further action needed." @@ -182,7 +182,7 @@ class HeatingControlRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, "simulation_config": simulation_config } ) @@ -228,8 +228,8 @@ class HeatingControlRecommender: description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " "temperature zone control)") - is_override = "heating_control" in self.property.override - if is_override: + already_installed = "heating_control" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) description = "Heating controls have already been upgraded, no further action needed." @@ -242,7 +242,7 @@ class HeatingControlRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, "simulation_config": simulation_config } ) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index d83b755e..432dc6a6 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -144,8 +144,8 @@ class HeatingRecommender: recommendation_description = f"{description} and {controls_description}" - is_override = "cavity_wall_insulation" in self.property.override - if is_override: + already_installed = "cavity_wall_insulation" in self.property.already_installed + if already_installed: total_costs = override_costs(total_costs) recommendation_description = "Heating system has already been upgraded, no further action needed." @@ -159,7 +159,7 @@ class HeatingRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **total_costs, "simulation_config": recommendation_simulation_config } @@ -371,8 +371,8 @@ class HeatingRecommender: n_rooms=self.property.number_of_rooms ) - is_override = "heating" in self.property.override - if is_override: + already_installed = "heating" in self.property.already_installed + if already_installed: boiler_costs = override_costs(boiler_costs) description = "Heating system has already been upgraded, no further action needed." @@ -386,7 +386,7 @@ class HeatingRecommender: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, "simulation_config": simulation_config, **boiler_costs } diff --git a/recommendations/HotwaterRecommendations.py b/recommendations/HotwaterRecommendations.py index 88cfa932..9c5c7045 100644 --- a/recommendations/HotwaterRecommendations.py +++ b/recommendations/HotwaterRecommendations.py @@ -42,8 +42,8 @@ class HotwaterRecommendations: recommendation_cost = self.costs.hot_water_tank_insulation() - is_override = "hot_water_tank_insulation" in self.property.override - if is_override: + already_installed = "hot_water_tank_insulation" in self.property.already_installed + if already_installed: recommendation_cost = override_costs(recommendation_cost) description = "Insulation tank has already been insulated, no further action required" else: @@ -60,7 +60,7 @@ class HotwaterRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **recommendation_cost, "simulation_config": {"hot_water_energy_eff_ending": "Average"} } diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 9e4c8e43..31720579 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -92,8 +92,8 @@ class LightingRecommendations: heat_demand_change, carbon_change = self.estimate_lighting_impact(number_non_lel_outlets) - is_override = "low_energy_lighting" in self.property.override - if is_override: + already_installed = "low_energy_lighting" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) description = "Low energy lighting has already been installed, no further action required" @@ -105,7 +105,7 @@ class LightingRecommendations: "description": description, "starting_u_value": None, "new_u_value": None, - "is_override": is_override, + "already_installed": already_installed, # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to # the proportion of lights that will be set to low energy "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2), diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 5ba7e82e..dc5ee7db 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -207,8 +207,8 @@ class RoofRecommendations: floor_area=self.property.insulation_floor_area, material=material ) - is_override = "loft_insulation" in self.property.override - if is_override: + already_installed = "loft_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) elif material["type"] == "flat_roof_insulation": cost_result = self.costs.flat_roof_insulation( @@ -216,8 +216,8 @@ class RoofRecommendations: material=material, non_insulation_materials=non_insulation_materials ) - is_override = "flat_roof_insulation" in self.property.override - if is_override: + already_installed = "flat_roof_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) else: raise ValueError("Invalid material type") @@ -238,7 +238,7 @@ class RoofRecommendations: "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **cost_result } ) diff --git a/recommendations/SecondaryHeating.py b/recommendations/SecondaryHeating.py index e426977e..5d763510 100644 --- a/recommendations/SecondaryHeating.py +++ b/recommendations/SecondaryHeating.py @@ -40,8 +40,8 @@ class SecondaryHeating: costs = self.costs.heater_removal(n_rooms=n_rooms) - is_override = "secondary_heating" in self.property.override - if is_override: + already_installed = "secondary_heating" in self.property.already_installed + if already_installed: costs = override_costs(costs) description = "Secondary heating system has already been removed, no further action required" else: @@ -56,7 +56,7 @@ class SecondaryHeating: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **costs, "simulation_config": { "secondheat_description_ending": "None" diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 72fcdf4b..58cf9735 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -111,8 +111,8 @@ class SolarPvRecommendations: description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p" f"anel system on {round(roof_coverage_percent)}% the roof.") - is_override = "solar_pv" in self.property.override - if is_override: + already_installed = "solar_pv" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) self.recommendation.append( @@ -124,7 +124,7 @@ class SolarPvRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **cost_result, # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale # back up here diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 7ffcda08..5b36bd9c 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -50,11 +50,11 @@ class VentilationRecommendations(Definitions): part = self.materials.copy() - is_override = "cavity_wall_insulation" in self.property.override + already_installed = "cavity_wall_insulation" in self.property.already_installed - estimated_cost = n_units * part[0]["cost"] if not is_override else 0 - labour_hours = 4 * n_units if not is_override else 0 - labour_days = 4 * n_units / 8.0 if not is_override else 0 + estimated_cost = n_units * part[0]["cost"] if not already_installed else 0 + labour_hours = 4 * n_units if not already_installed else 0 + labour_days = 4 * n_units / 8.0 if not already_installed else 0 part[0]["total"] = estimated_cost part[0]["quantity"] = n_units @@ -69,7 +69,7 @@ class VentilationRecommendations(Definitions): "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, - "is_override": is_override, + "already_installed": already_installed, "sap_points": 0, "heat_demand": 0, "adjusted_heat_demand": 0, diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 471a62cb..feb2620b 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -221,8 +221,8 @@ class WallRecommendations(Definitions): material=material.to_dict(), ) - is_override = "cavity_wall_insulation" in self.property.override - if is_override: + already_installed = "cavity_wall_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) recommendations.append( @@ -241,7 +241,7 @@ class WallRecommendations(Definitions): "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **cost_result } ) @@ -282,8 +282,8 @@ class WallRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) - is_override = "internal_wall_insulation" in self.property.override - if is_override: + already_installed = "internal_wall_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) elif material["type"] == "external_wall_insulation": @@ -292,8 +292,8 @@ class WallRecommendations(Definitions): material=material.to_dict(), non_insulation_materials=non_insulation_materials ) - is_override = "external_wall_insulation" in self.property.override - if is_override: + already_installed = "external_wall_insulation" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) else: raise ValueError("Invalid material type") @@ -313,7 +313,7 @@ class WallRecommendations(Definitions): "description": self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, - "is_override": is_override, + "already_installed": already_installed, "sap_points": None, **cost_result } diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index b2fe20a6..b7c2823a 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -71,8 +71,8 @@ class WindowsRecommendations: is_secondary_glazing=is_secondary_glazing ) - is_override = "windows_glazing" in self.property.override - if is_override: + already_installed = "windows_glazing" in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) description = "The property already has double glazing installed. No further action is required." else: @@ -98,7 +98,7 @@ class WindowsRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, - "is_override": is_override, + "already_installed": already_installed, **cost_result, "is_secondary_glazing": is_secondary_glazing } From c58389a26695d863d003a4cf2c9f26515f9898ea Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 14 Apr 2024 14:57:46 +0100 Subject: [PATCH 40/44] updated push to db --- backend/app/db/functions/recommendations_functions.py | 2 +- backend/app/db/models/recommendations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 43daec77..b22ce92f 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -86,7 +86,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "total_work_hours": rec["labour_hours"], "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], - "is_override": rec["is_override"], + "already_installed": rec["already_installed"], } for rec in recommendations_to_upload ] diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index be5ff30c..186f87a8 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -30,7 +30,7 @@ class Recommendation(Base): rental_yield_increase = Column(Float) total_work_hours = Column(Float) labour_days = Column(Float) - is_override = Column(Boolean, nullable=False, default=False) + already_installed = Column(Boolean, nullable=False, default=False) class RecommendationMaterials(Base): From f1e3bca9bff0c68ba9ce068c91a91268da794cb0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 14 Apr 2024 14:59:30 +0100 Subject: [PATCH 41/44] updated asset list for immo to reference already installed --- etl/customers/immo/pilot/asset_list.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index 07ebe884..d8839924 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -48,7 +48,7 @@ patches = [ # This is information that is found as a result of the non-invasives, that mean that certain measures # have been installed already. To reflect this in the front end, it is included in the recommendation, however # the cost is removed and instead, a message is presented saying that the measure is already installed. -overrides = [ +already_installed = [ { 'address': '5 Oaklands', 'postcode': 'B62 0JA', @@ -87,11 +87,11 @@ def app(): ) # Store overrides in s3 - overrides_filename = f"{USER_ID}/{PORTFOLIO_ID}/overrides.json" + already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json" save_csv_to_s3( - dataframe=pd.DataFrame(overrides), + dataframe=pd.DataFrame(already_installed), bucket_name="retrofit-plan-inputs-dev", - file_name=overrides_filename + file_name=already_installed_filename ) # Store patches in s3 @@ -109,7 +109,7 @@ def app(): "goal": "Increase EPC", "goal_value": "C", "trigger_file_path": filename, - "overrides_file_path": overrides_filename, + "already_installed_file_path": already_installed_filename, "patches_file_path": patches_filename, "budget": None, } @@ -122,7 +122,7 @@ def app(): "goal": "Increase EPC", "goal_value": "B", "trigger_file_path": filename, - "overrides_file_path": overrides_filename, + "already_installed_file_path": already_installed_filename, "patches_file_path": patches_filename, "budget": None, } From 046ac3dc39bc7c478a91fcaa58bddc30508c5166 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 14 Apr 2024 15:05:31 +0100 Subject: [PATCH 42/44] fixed bug in already installed --- etl/customers/immo/pilot/asset_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/customers/immo/pilot/asset_list.py b/etl/customers/immo/pilot/asset_list.py index d8839924..e587cc25 100644 --- a/etl/customers/immo/pilot/asset_list.py +++ b/etl/customers/immo/pilot/asset_list.py @@ -52,7 +52,7 @@ already_installed = [ { 'address': '5 Oaklands', 'postcode': 'B62 0JA', - "overrides": ["windows_glazing"] + "already_installed": ["windows_glazing"] } ] From 56bf3c121fbc0d4bb31a5e1b073b80daac7dba51 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 15 Apr 2024 13:31:46 +0100 Subject: [PATCH 43/44] Adding cdn to terraform --- infrastructure/terraform/main.tf | 9 +++ .../terraform/modules/cloudfront/main.tf | 65 +++++++++++++++++++ .../terraform/modules/cloudfront/variables.tf | 9 +++ 3 files changed, 83 insertions(+) create mode 100644 infrastructure/terraform/modules/cloudfront/main.tf create mode 100644 infrastructure/terraform/modules/cloudfront/variables.tf diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index d545cdf8..1d0562dd 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -181,4 +181,13 @@ module "lambda_carbon_prediction_ecr" { module "lambda_heat_prediction_ecr" { ecr_name = "lambda-heat-prediction-${var.stage}" source = "./modules/ecr" +} + +############################################## +# CDN - Cloudfront +############################################## +module "cloudfront_distribution" { + source = "./modules/cloudfront" + bucket_name = module.s3.bucket_name + stage = var.stage } \ No newline at end of file diff --git a/infrastructure/terraform/modules/cloudfront/main.tf b/infrastructure/terraform/modules/cloudfront/main.tf new file mode 100644 index 00000000..fbb88160 --- /dev/null +++ b/infrastructure/terraform/modules/cloudfront/main.tf @@ -0,0 +1,65 @@ +resource "aws_cloudfront_distribution" "s3_distribution" { + origin { + domain_name = "${aws_s3_bucket.bucket.bucket_regional_domain_name}" + origin_id = "S3-${var.bucket_name}" + + s3_origin_config { + origin_access_identity = aws_cloudfront_origin_access_identity.oai.cloudfront_access_identity_path + } + } + + enabled = true + + default_cache_behavior { + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + target_origin_id = "S3-${var.bucket_name}" + viewer_protocol_policy = "redirect-to-https" + compress = true + + forwarded_values { + query_string = false + cookies { + forward = "none" + } + } + + min_ttl = 0 + default_ttl = 86400 + max_ttl = 31536000 + } + + price_class = "PriceClass_All" + + restrictions { + geo_restriction { + restriction_type = "none" + } + } + + viewer_certificate { + cloudfront_default_certificate = true + } +} + +resource "aws_cloudfront_origin_access_identity" "oai" { + comment = "OAI for ${var.bucket_name}" +} + +resource "aws_s3_bucket_policy" "bucket_policy" { + bucket = aws_s3_bucket.bucket.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + AWS = "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity ${aws_cloudfront_origin_access_identity.oai.id}" + } + Action = "s3:GetObject" + Resource = "${aws_s3_bucket.bucket.arn}/*" + }, + ] + }) +} diff --git a/infrastructure/terraform/modules/cloudfront/variables.tf b/infrastructure/terraform/modules/cloudfront/variables.tf new file mode 100644 index 00000000..433edc24 --- /dev/null +++ b/infrastructure/terraform/modules/cloudfront/variables.tf @@ -0,0 +1,9 @@ +variable "bucket_name" { + description = "The name of the bucket" + type = string +} + +variable "stage" { + description = "The deployment stage" + type = string +} From ce546b56f7db4a88d82ee3f72148d2b4fe64f1c2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 15 Apr 2024 13:38:23 +0100 Subject: [PATCH 44/44] passing additional data to cloudfront distribution --- infrastructure/terraform/main.tf | 9 ++++++--- .../terraform/modules/cloudfront/main.tf | 6 +++--- .../terraform/modules/cloudfront/variables.tf | 15 +++++++++++++++ infrastructure/terraform/modules/s3/outputs.tf | 12 ++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 1d0562dd..fde25487 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -187,7 +187,10 @@ module "lambda_heat_prediction_ecr" { # CDN - Cloudfront ############################################## module "cloudfront_distribution" { - source = "./modules/cloudfront" - bucket_name = module.s3.bucket_name - stage = var.stage + source = "./modules/cloudfront" + bucket_name = module.s3.bucket_name + bucket_id = module.s3.bucket_id + bucket_arn = module.s3.bucket_arn + bucket_domain_name = module.s3.bucket_domain_name + stage = var.stage } \ No newline at end of file diff --git a/infrastructure/terraform/modules/cloudfront/main.tf b/infrastructure/terraform/modules/cloudfront/main.tf index fbb88160..281ff09f 100644 --- a/infrastructure/terraform/modules/cloudfront/main.tf +++ b/infrastructure/terraform/modules/cloudfront/main.tf @@ -1,6 +1,6 @@ resource "aws_cloudfront_distribution" "s3_distribution" { origin { - domain_name = "${aws_s3_bucket.bucket.bucket_regional_domain_name}" + domain_name = var.bucket_domain_name origin_id = "S3-${var.bucket_name}" s3_origin_config { @@ -47,7 +47,7 @@ resource "aws_cloudfront_origin_access_identity" "oai" { } resource "aws_s3_bucket_policy" "bucket_policy" { - bucket = aws_s3_bucket.bucket.id + bucket = var.bucket_id policy = jsonencode({ Version = "2012-10-17" @@ -58,7 +58,7 @@ resource "aws_s3_bucket_policy" "bucket_policy" { AWS = "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity ${aws_cloudfront_origin_access_identity.oai.id}" } Action = "s3:GetObject" - Resource = "${aws_s3_bucket.bucket.arn}/*" + Resource = "${var.bucket_arn}/*" }, ] }) diff --git a/infrastructure/terraform/modules/cloudfront/variables.tf b/infrastructure/terraform/modules/cloudfront/variables.tf index 433edc24..88f770a8 100644 --- a/infrastructure/terraform/modules/cloudfront/variables.tf +++ b/infrastructure/terraform/modules/cloudfront/variables.tf @@ -7,3 +7,18 @@ variable "stage" { description = "The deployment stage" type = string } + +variable "bucket_id" { + description = "The ID of the S3 bucket" + type = string +} + +variable "bucket_arn" { + description = "The ARN of the S3 bucket" + type = string +} + +variable "bucket_domain_name" { + description = "The regional domain name of the S3 bucket" + type = string +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/s3/outputs.tf b/infrastructure/terraform/modules/s3/outputs.tf index a5e7ddb4..7668dbc4 100644 --- a/infrastructure/terraform/modules/s3/outputs.tf +++ b/infrastructure/terraform/modules/s3/outputs.tf @@ -2,3 +2,15 @@ output "bucket_name" { description = "The name of the S3 bucket" value = aws_s3_bucket.bucket.bucket } + +output "bucket_id" { + value = aws_s3_bucket.bucket.id +} + +output "bucket_arn" { + value = aws_s3_bucket.bucket.arn +} + +output "bucket_domain_name" { + value = aws_s3_bucket.bucket.bucket_regional_domain_name +} \ No newline at end of file