From d703447c6a9ee2b600ce5c1949f864903adde016 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 15:39:51 +0000 Subject: [PATCH 01/27] Adding new materials to Materials enum --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 3 + backend/app/plan/temp_script_for_flight.py | 176 --------------------- recommendations/RoofRecommendations.py | 17 +- 5 files changed, 12 insertions(+), 188 deletions(-) delete mode 100644 backend/app/plan/temp_script_for_flight.py diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 1dc47276..812c1ebb 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -15,6 +15,9 @@ class MaterialType(enum.Enum): cavity_wall_insulation = "cavity_wall_insulation" mechanical_ventilation = "mechanical_ventilation" loft_insulation = "loft_insulation" + exposed_floor_insulation = "exposed_floor_insulation" + flat_roof_insulation = "flat_roof_insulation" + room_roof_insulation = "room_roof_insulation" class DepthUnit(enum.Enum): diff --git a/backend/app/plan/temp_script_for_flight.py b/backend/app/plan/temp_script_for_flight.py deleted file mode 100644 index 9170b4c1..00000000 --- a/backend/app/plan/temp_script_for_flight.py +++ /dev/null @@ -1,176 +0,0 @@ -from datetime import datetime - -import pandas as pd -from epc_api.client import EpcClient -from fastapi import APIRouter, Depends -from sqlalchemy.exc import IntegrityError, OperationalError -from sqlalchemy.orm import sessionmaker -from starlette.responses import Response - -from backend.app.config import get_settings -from backend.app.db.connection import db_engine -from backend.app.db.functions.materials_functions import get_materials -from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations -from backend.app.db.functions.property_functions import ( - create_property, create_property_details_epc, create_property_targets, update_property_data -) -from backend.app.db.functions.recommendations_functions import ( - create_plan, create_plan_recommendations, upload_recommendations -) -from backend.app.db.models.portfolio import rating_lookup -from backend.app.dependencies import validate_token -from backend.app.plan.schemas import PlanTriggerRequest -from backend.app.plan.utils import ( - create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id -) -from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 - -from backend.ml_models.sap_change_model.api import SAPChangeModelAPI -from backend.Property import Property -from etl.epc.DataProcessor import DataProcessor -from etl.epc.settings import COLUMNS_TO_MERGE_ON -from recommendations.FloorRecommendations import FloorRecommendations -from recommendations.optimiser.CostOptimiser import CostOptimiser -from recommendations.optimiser.GainOptimiser import GainOptimiser -from recommendations.optimiser.optimiser_functions import prepare_input_measures -from recommendations.WallRecommendations import WallRecommendations -from utils.logger import setup_logger -from utils.s3 import read_dataframe_from_s3_parquet - -logger = setup_logger() - -import pickle - -with open('local_data.pickle', 'rb') as f: - local_data = pickle.load(f) - -with open("property_dimensions.pickle", "rb") as f: - property_dimensions = pickle.load(f) - -with open("sap_change_dataset.pickle", "rb") as f: - sap_change_dataset = pickle.load(f) - -created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - -plan_input = local_data["plan_input"] -uprn_filenames = local_data["uprn_filenames"] -local_property_data = local_data["local_property_data"] -materials = local_data["materials"] -materials_by_type = filter_materials(materials) -cleaned = local_data["cleaned"] -cleaning_data = local_data["cleaning_data"] - -# Need to find some proper materials -materials_by_type["walls"] += [ - {'id': 4, 'type': 'cavity_wall_insulation', 'description': 'Example Material 1', - 'depths': None, - 'depth_unit': None, 'cost': 20, - 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.0278, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.036, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': None, 'created_at': None, 'is_active': True}, - {'id': 10, 'type': "cavity_wall_insulation", 'description': 'Example Material 2', - 'depths': None, 'depth_unit': None, 'cost': 25, 'cost_unit': 'gbp_sq_meter', - 'r_value_per_mm': 0.02631579, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': None, - 'created_at': None, 'is_active': True} -] - -epc_client = EpcClient(auth_token="NO-TOKEN") - -input_properties = [] -for i, config in enumerate(plan_input): - property_id = local_property_data[i]["id"] - input_properties.append( - Property( - postcode=config['postcode'], - address1=config['address'], - epc_client=epc_client, - id=property_id - ) - ) - -logger.info("Getting EPC, and spatial data") -for i, p in enumerate(input_properties): - p.data = local_property_data[i]["data"] - p.uprn = local_property_data[i]["uprn"] - p.id = local_property_data[i]["id"] - p.full_sap_epc = local_property_data[i]["full_sap_epc"] - p.old_data = local_property_data[i]["old_data"] - p.is_listed = False - p.in_conservation_area = False - p.is_heritage = False - - p.set_year_built() - - # TODO: TESTING - p.data['number-habitable-rooms'] = 3 - -recommendations = {} -recommendations_scoring_data = [] - -for p in input_properties: - property_recommendations = [] - - # Property recommendations - p.get_components(cleaned) - - # Floor recommendations - floor_recommender = FloorRecommendations( - property_instance=p, - materials=materials_by_type["floor"], - ) - floor_recommender.recommend() - - if floor_recommender.recommendations: - property_recommendations.append(floor_recommender.recommendations) - - # Wall recommendations - - wall_recomender = WallRecommendations( - property_instance=p, - materials=materials_by_type["walls"] - ) - wall_recomender.recommend() - - if wall_recomender.recommendations: - property_recommendations.append(wall_recomender.recommendations) - - # We insert temporary ids into the recommendations which is important for the optimiser later - property_recommendations = insert_temp_recommendation_id(property_recommendations) - - if not property_recommendations: - continue - - recommendations[p.id] = property_recommendations - - # Finally, we'll prepare data for predicting the impact on SAP - # TODO: We should use the cleaned data from get_components in the data rather than the raw - # values. We should create a method in Property which takes the EPC data and inserts the cleaned - # data - - data_processor = DataProcessor(None, newdata=True) - data_processor.insert_data(pd.DataFrame([p.data.copy()])) - data_processor.pre_process() - - starting_epc_data = data_processor.get_component_features(suffix="_STARTING") - ending_epc_data = data_processor.get_component_features(suffix="_ENDING") - fixed_data = data_processor.get_fixed_features() - - # We update the ending record with the recommended updates and we set lodgement date to today - ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at - - for recommendations_by_type in property_recommendations: - for rec in recommendations_by_type: - scoring_dict = create_recommendation_scoring_data( - property=p, - recommendation=rec, - starting_epc_data=starting_epc_data, - ending_epc_data=ending_epc_data, - fixed_data=fixed_data, - ) - - recommendations_scoring_data.append(scoring_dict) - -# cleanup -del data_processor diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 283370ac..e5200904 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -47,7 +47,8 @@ class RoofRecommendations: # Building regulations part L recommend installing at least 270mm of insulation, however generally we # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation - if insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM: + # This only holds true for pitched roofs + if (insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]: return # If we have a u-value already, need to implement this @@ -61,7 +62,7 @@ class RoofRecommendations: return if self.property.roof["is_roof_room"]: - self.recommend_room_roof_insulation(u_value, insulation_thickness) + self.recommend_room_roof_insulation(u_value) return raise NotImplementedError("Implement me") @@ -125,8 +126,9 @@ class RoofRecommendations: for depth, cost_per_unit in zip(material["depths"], material["cost"]): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the - # loft is already partially insulated - if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: + # loft is already partially insulated. + # Note: This requirement is only for loft insulation + if ((depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: continue part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) @@ -178,7 +180,7 @@ class RoofRecommendations: self.recommendations = recommendations - def recommend_room_roof_insulation(self, u_value, insulation_thickness): + def recommend_room_roof_insulation(self, u_value): """ This method recommends room in roof insulation for properties that have been identified to possess a room in roof. @@ -217,7 +219,6 @@ class RoofRecommendations: - Flat ceilings can be insulated like a standard loft. :param u_value: Current u-value of the roof - :param insulation_thickness: Current insulation thickness of the roof :return: """ @@ -232,10 +233,6 @@ class RoofRecommendations: recommendations = [] for material in roof_roof_insulation_materials: for depth, cost_per_unit in zip(material["depths"], material["cost"]): - # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the - # loft is already partially insulated - if (depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM: - continue part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) From eed5a0baa358fd55c4b063e394ada697ce9a9dd6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 16:00:00 +0000 Subject: [PATCH 02/27] added property get_components template, when there is no component --- backend/Property.py | 8 ++++- backend/app/plan/router.py | 30 ------------------- backend/app/plan/utils.py | 2 +- recommendations/RoofRecommendations.py | 10 +++++++ recommendations/recommendation_utils.py | 3 ++ .../tests/test_roof_recommendations.py | 23 ++++++++++++++ 6 files changed, 44 insertions(+), 32 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 1094e7b2..2359ea6a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -288,10 +288,16 @@ class Property(Definitions): for description, attribute in cleaned.items(): if self.data[description] in self.DATA_ANOMALY_MATCHES: + template = cleaned[description][0] + fill_dict = dict(zip(template.keys(), [None] * len(template))) + fill_dict.update({ + "original_description": self.data[description], + "clean_description": self.data[description], + }) setattr( self, self.ATTRIBUTE_MAP[description], - {"original_description": self.data[description], "clean_description": self.data[description]} + fill_dict, ) continue diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 23ad4262..e531896e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -39,7 +39,6 @@ from recommendations.optimiser.optimiser_functions import prepare_input_measures from recommendations.WallRecommendations import WallRecommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet -from tqdm import tqdm logger = setup_logger() @@ -122,35 +121,6 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - # import pickle - # with open("input_properties.pickle", "rb") as f: - # input_properties = pickle.load(f) - # - # import pickle - # with open("new_sap_dataset.pickle", "rb") as f: - # new_sap_dataset = pickle.load(f) - - # import pickle - # with open("cleaned.pickle", "rb") as f: - # cleaned = pickle.load(f) - - # with open("sap_dataset.pickle", "rb") as f: - # sap_dataset = pickle.load(f) - - # with open("materials_by_type", "rb") as f: - # materials_by_type = pickle.load(f) - - # materials_by_type["floor"].append( - # {'id': 18, 'type': 'exposed_floor_insulation', 'description': 'Rockwool Stone Wool insulation', - # 'depths': [50, 100, 140], 'depth_unit': 'mm', 'cost': [8, 11, 15], - # 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.026315789473684213, - # 'r_value_unit': 'square_meter_kelvin_per_watt', - # 'thermal_conductivity': 0.038, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - # 'link': 'https://insulation4less.co.uk/products/rockwool-flexi-slab-all-sizes?variant=33409590853685', - # 'created_at': datetime(2023, 8, 10, 16, 59, 10, 815531), 'is_active': True} - # - # ) - recommendations = {} recommendations_scoring_data = [] diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 71a61be1..36e90d61 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -17,7 +17,7 @@ def filter_materials(materials): "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"], "ventilation": ["mechanical_ventilation"], - "roof": ["loft_insulation"] + "roof": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"] } materials = [row2dict(material) for material in materials] diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index e5200904..bfa63908 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -36,6 +36,10 @@ class RoofRecommendations: self.materials = materials def recommend(self): + + if self.property.roof["has_dwelling_above"]: + return + u_value = self.property.roof["thermal_transmittance"] insulation_thickness = convert_thickness_to_numeric( @@ -53,6 +57,12 @@ class RoofRecommendations: # If we have a u-value already, need to implement this if u_value: + if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + # The floor is already compliant + return + + if self.property.data["transaction-type"] == "new dwelling": + return raise NotImplementedError("Implement me") u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 13f58fd9..063a274c 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -585,6 +585,9 @@ def convert_thickness_to_numeric(string_thickness, is_pitched): :return: integer measure of insulation thickness """ + if string_thickness is None: + return 0 + if is_pitched: lookup = { "none": 0, diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 37cc2daf..551407da 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -427,3 +427,26 @@ class TestRoofRecommendations: assert roof_recommender13.recommendations[0]["description"] == \ "Insulate the home's flat roof with 220mm of Example flat roof insulation" + + def test_property_above(self): + property_instance14 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) + property_instance14.age_band = "F" + property_instance14.floor_area = 100 + property_instance14.roof = { + 'original_description': '(other premises above)', + 'clean_description': '(other premises above)', 'thermal_transmittance': 0, + 'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False, + 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': False, 'has_dwelling_above': True, 'is_valid': True, + 'insulation_thickness': None + } + + roof_recommender14 = RoofRecommendations( + property_instance=property_instance14, materials=loft_insulation_materials + ) + + assert not roof_recommender14.recommendations + + roof_recommender14.recommend() + + assert not roof_recommender14.recommendations From beb45d451d950f38e20900a7701d76ab8fabc38d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 16:52:47 +0000 Subject: [PATCH 03/27] fetch old data construction age band if missing --- backend/Property.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 2359ea6a..1cebc4a9 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1,4 +1,5 @@ from datetime import datetime +from collections import Counter import re import os import pandas as pd @@ -334,8 +335,24 @@ class Property(Definitions): raise ValueError("Property does not contain data") self.construction_age_band = DataProcessor.clean_construction_age_band(self.data["construction-age-band"]) + if self.construction_age_band in self.DATA_ANOMALY_MATCHES: + if self.old_data: + # Take the most recent + max_datetime = max( + [x["lodgement-datetime"] for x in self.old_data if + x["construction-age-band"] not in self.DATA_ANOMALY_MATCHES] + ) + most_recent = [x for x in self.old_data if x["lodgement-datetime"] == max_datetime] + + self.construction_age_band = DataProcessor.clean_construction_age_band( + most_recent[0]["construction-age-band"] + ) + self.age_band = england_wales_age_band_lookup.get(self.construction_age_band) + if self.age_band is None: + raise ValueError("age_band is missing") + def set_spatial(self, spatial: pd.DataFrame): """ Sets whether the property is in a conservation area given the output of the ConservationAreaClient From be9a9601862165cf0582237dba0976fb8292f0e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 17 Nov 2023 17:03:14 +0000 Subject: [PATCH 04/27] Added default age band for new builds --- backend/Property.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 1cebc4a9..259ca724 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -350,6 +350,10 @@ class Property(Definitions): self.age_band = england_wales_age_band_lookup.get(self.construction_age_band) + if (self.data["transaction-type"] == "new dwelling") and (self.age_band is None): + self.age_band = "L" + self.construction_age_band = 'England and Wales: 2012 onwards' + if self.age_band is None: raise ValueError("age_band is missing") From 76a03cdf23189146e70ed27727f4960b5a0f08ef Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 11:12:34 +0000 Subject: [PATCH 05/27] Final fixes to get portfolio working --- backend/Property.py | 1 + backend/app/plan/router.py | 13 ++++++++++--- etl/epc/DataProcessor.py | 12 +++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 259ca724..370eca06 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -749,6 +749,7 @@ class Property(Definitions): "TOTAL_FLOOR_AREA": self.floor_area, **epc_raw_data, "BUILT_FORM": built_form, + "POSTCODE": self.data["postcode"], } return property_data diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e531896e..4064452f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -211,11 +211,18 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Preparing data for scoring in sap change api") recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) - # Perform the same cleaning as in the model + # Perform the same cleaning as in the model - first clean number of room variables though recommendations_scoring_data = DataProcessor.apply_averages_cleaning( data_to_clean=recommendations_scoring_data, cleaning_data=cleaning_data, - cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"] + cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'], + colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"], + ) + + recommendations_scoring_data = DataProcessor.apply_averages_cleaning( + data_to_clean=recommendations_scoring_data, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"], ).drop(columns=["LOCAL_AUTHORITY"]) recommendations_scoring_data = DataProcessor.clean_missings_after_description_process( @@ -303,7 +310,7 @@ async def trigger_plan(body: PlanTriggerRequest): # 3) the recommendations logger.info("Uploading recommendations to the database") - for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): + for i in range(0, len(input_properties), BATCH_SIZE): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 3ef485b8..0587fdbe 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -179,7 +179,6 @@ class DataProcessor: # We have some non-standard construction age bands which we'll clean for matching if not self.newdata: self.standardise_construction_age_band() - self.clean_missing_rooms() self.recast_df_columns( @@ -451,7 +450,7 @@ class DataProcessor: self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0) @staticmethod - def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on): + def apply_averages_cleaning(data_to_clean, cleaning_data, cols_to_merge_on, colnames=None): """ Clean the input DataFrame using averages from a cleaning DataFrame. @@ -459,11 +458,16 @@ class DataProcessor: :param cleaning_data: DataFrame containing data for cleaning. :param cols_to_merge_on: Columns on which merging is based. We pass cols_to_merge_on to this function as this differs depending on where the function is being used. + :param colnames: If specified can be used to state exactly which columns to clean :return: Cleaned DataFrame. """ + # The desired colnames to clean - which may not be present + if colnames is None: + colnames = ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"] + cols_to_clean = [ - c for c in ["TOTAL_FLOOR_AREA", "FLOOR_HEIGHT", "FIXED_LIGHTING_OUTLETS_COUNT"] if + c for c in colnames if c in data_to_clean.columns ] @@ -492,6 +496,8 @@ class DataProcessor: for col in cols_to_clean: data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"], inplace=True) data_to_clean.drop(columns=[f"{col}_AVERAGE"], inplace=True) + # If we still have missings + data_to_clean[col].fillna(data_to_clean[col].mean(), inplace=True) return data_to_clean From 3d0ebbb24ee108a81fa83f6c9daeda4a30117091 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 14:42:16 +0000 Subject: [PATCH 06/27] completed build of new demo portfolio - some fixed required still --- backend/app/plan/router.py | 11 +++++++++-- datatypes/enums.py | 1 + recommendations/VentilationRecommendations.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 4064452f..ff56aa38 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -73,7 +73,8 @@ async def trigger_plan(body: PlanTriggerRequest): input_properties = [] for config in plan_input: # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly - # TODO: implment validation + # TODO: implment validation. We should also standardise postcode and address in some fashion as + # a postcode of abcdef would be considered different to ABCDEF # Create a record in db property_id, is_new = create_property( session, portfolio_id=body.portfolio_id, address=config['address'], postcode=config['postcode'] @@ -310,7 +311,8 @@ async def trigger_plan(body: PlanTriggerRequest): # 3) the recommendations logger.info("Uploading recommendations to the database") - for i in range(0, len(input_properties), BATCH_SIZE): + session.commit() + for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] @@ -323,6 +325,11 @@ async def trigger_plan(body: PlanTriggerRequest): ) create_property_details_epc(session, property_details_epc) + # TODO: TEMP + if p.data["uprn"] == "": + print("Get rid of me!") + p.data["uprn"] = 0 + property_data = p.get_full_property_data() update_property_data( session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data diff --git a/datatypes/enums.py b/datatypes/enums.py index 1b0959e0..31f094ad 100644 --- a/datatypes/enums.py +++ b/datatypes/enums.py @@ -3,3 +3,4 @@ import enum class QuantityUnits(enum.Enum): m2 = "m2" + part = "part" diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 35de9b3b..a639905b 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -54,7 +54,7 @@ class VentilationRecommendations(Definitions): part[0]["estimated_cost"] = estimated_cost part[0]["quantity"] = n_units - part[0]["quantity_unit"] = None + part[0]["quantity_unit"] = "part" # We recommend installing two mechanical ventilation systems self.recommendation = [ From 2cc5a8f4652d9a06b3f51b8cd65a9633bb47aa86 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 20 Nov 2023 16:22:51 +0000 Subject: [PATCH 07/27] Adding the clear_portfolio method for quickly emptying out a portfolio for easy rebuilds --- .../db/functions/recommendations_functions.py | 53 ++++++++++++++++--- backend/app/plan/router.py | 2 +- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index b9ec6fc3..0bdf69ce 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -1,10 +1,14 @@ -from sqlalchemy import insert +from sqlalchemy import insert, delete +from sqlalchemy.orm import Session from backend.app.db.models.recommendations import Plan, Recommendation, RecommendationMaterials, PlanRecommendations +from backend.app.db.models.portfolio import PropertyModel, PropertyTargetsModel, PropertyDetailsMeter, \ + PropertyDetailsEpcModel -def create_plan(session, plan): +def create_plan(session: Session, plan): """ This function will create a record for the plan in the database if it does not exist. + :param session: The database session :param plan: dictionary of data representing a plan to be created """ @@ -15,7 +19,7 @@ def create_plan(session, plan): return new_plan.id -def create_recommendation(session, recommendation): +def create_recommendation(session: Session, recommendation): """ This function will create a record for the recommendation in the database if it does not exist. :param session: The database session @@ -29,7 +33,7 @@ def create_recommendation(session, recommendation): return new_recommendation.id -def create_recommendation_material(session, recommendation_id, material_id, depth): +def create_recommendation_material(session: Session, recommendation_id, material_id, depth): """ This function will create a record for the recommendation_material in the database if it does not exist. :param session: The databse session @@ -49,9 +53,10 @@ def create_recommendation_material(session, recommendation_id, material_id, dept return new_recommendation_material.id -def create_plan_recommendations(session, plan_id, recommendation_ids): +def create_plan_recommendations(session: Session, plan_id, recommendation_ids): """ This function will create records for the plan_recommendation in the database. + :param session: The database session :param plan_id: ID of the plan :param recommendation_ids: list of recommendation IDs """ @@ -63,7 +68,7 @@ def create_plan_recommendations(session, plan_id, recommendation_ids): session.execute(insert(PlanRecommendations).values(data)) -def upload_recommendations(session, recommendations_to_upload, property_id): +def upload_recommendations(session: Session, recommendations_to_upload, property_id): # Prepare data for bulk insert for Recommendation recommendations_data = [ { @@ -112,3 +117,39 @@ def upload_recommendations(session, recommendations_to_upload, property_id): session.flush() return uploaded_recommendation_ids + + +def clear_portfolio(session: Session, portfolio_id: int): + # Fetch all property IDs associated with the given portfolio + property_ids = session.query(PropertyModel.id).filter(PropertyModel.portfolio_id == portfolio_id).all() + property_ids = [p.id for p in property_ids] + + # Fetch all recommendation IDs associated with the properties + recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all() + recommendation_ids = [r.id for r in recommendation_ids] + + # Delete all entries from RecommendationMaterials for these recommendations + session.execute( + delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) + ) + + # Delete all entries from PlanRecommendations that reference plans in the portfolio + session.execute(delete(PlanRecommendations).where(PlanRecommendations.plan_id.in_( + session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar() + ))) + + # Delete all Plans associated with the portfolio + session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id)) + + # Delete all Recommendations associated with the properties + session.execute(delete(Recommendation).where(Recommendation.property_id.in_(property_ids))) + + # Now, delete the PropertyModels and related details + # Delete PropertyTargetsModel, PropertyDetailsMeter, PropertyDetailsEpcModel, and PropertyModel + session.execute(delete(PropertyTargetsModel).where(PropertyTargetsModel.portfolio_id == portfolio_id)) + # session.execute(delete(PropertyDetailsMeter).where(PropertyDetailsMeter.uprn.in_(property_ids))) + session.execute(delete(PropertyDetailsEpcModel).where(PropertyDetailsEpcModel.portfolio_id == portfolio_id)) + session.execute(delete(PropertyModel).where(PropertyModel.portfolio_id == portfolio_id)) + + # Commit the changes + session.commit() diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index ff56aa38..83a57d07 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -312,7 +312,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Uploading recommendations to the database") session.commit() - for i in tqdm(range(0, len(input_properties), BATCH_SIZE)): + for i in range(0, len(input_properties), BATCH_SIZE): try: # Take a slice of the input_properties list to make a batch batch_properties = input_properties[i:i + BATCH_SIZE] From a74459bf46b171f37542af3e60ab3498a3822f89 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 11:53:22 +0000 Subject: [PATCH 08/27] Creating new cost class to handle new costing data --- backend/app/plan/utils.py | 2 +- recommendations/Costs.py | 139 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 recommendations/Costs.py diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 36e90d61..e2bf9d86 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -173,7 +173,7 @@ def create_recommendation_scoring_data( parts = recommendation["parts"] if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - + scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: diff --git a/recommendations/Costs.py b/recommendations/Costs.py new file mode 100644 index 00000000..efa22fd8 --- /dev/null +++ b/recommendations/Costs.py @@ -0,0 +1,139 @@ +class Costs: + """ + A class to calculate the costs associated with construction works, + specifically focusing on cavity wall insulation. + It includes contingency, preliminaries, profit margin, and VAT calculations. + + As a sense check, there is a useful article from checkatrade on retrofitting walls and expected costs: + https://www.checkatrade.com/blog/cost-guides/retrofit-insulation-cost/ + """ + + # Contingency is a percentage of the total cost of the work and covers unforseen expenses + # We assume a conservative 10% contingency for all works which is a rate defined by SPONs + CONTINGENCY = 0.1 + + # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs + # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% + # rate, on the total cost before VAT, as recommended by SPONs + PRELIMINARIES = 0.1 + + VAT_RATE = 0.2 + PROFIT_MARGIN = 0.15 + + def __init__(self, property_instance): + """ + Initializes the Costs class with a property instance. + + :param property_instance: Instance of a Property class containing relevant details like wall area. + """ + if not hasattr(property_instance, 'insulation_wall_area'): + raise ValueError("Property instance must have an 'insulation_wall_area' attribute") + self.property = property_instance + + def cavity_wall_insulation(self, material): + """ + Calculates the total cost for cavity wall insulation based on material and labor costs, + including contingency, preliminaries, profit, and VAT. + + :return: A dictionary containing detailed cost breakdown. + """ + # Cost per m2 + material = { + "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " + "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " + "retaining discs", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 2.25, + "labour_hours": 0.13 + } + + material_cost_per_m2 = material["material_cost"] + wall_area = self.property.insulation_wall_area + + # This is the amount of material required in m3, assuming a standard 75mm depth + volume = 0.075 * wall_area + + base_material_cost = material_cost_per_m2 * wall_area + labour_cost = material["labour_cost"] * wall_area + + subtotal_before_profit = base_material_cost + labour_cost + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + labour_hours = material["labour_hours"] * wall_area + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": base_material_cost, + "profit": profit_cost, + "labour_hours": labour_hours + } + + def cavity_wall_insulation(self, material): + """ + Calculates the total cost for cavity wall insulation based on material and labor costs, + including contingency, preliminaries, profit, and VAT. + + :return: A dictionary containing detailed cost breakdown. + """ + # Cost per m2 + material = { + "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " + "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " + "retaining discs", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 2.25, + "labour_hours": 0.13 + } + + material_cost_per_m2 = material["material_cost"] + wall_area = self.property.insulation_wall_area + + # This is the amount of material required in m3, assuming a standard 75mm depth + volume = 0.075 * wall_area + + base_material_cost = material_cost_per_m2 * wall_area + labour_cost = material["labour_cost"] * wall_area + + subtotal_before_profit = base_material_cost + labour_cost + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + labour_hours = material["labour_hours"] * wall_area + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": base_material_cost, + "profit": profit_cost, + "labour_hours": labour_hours + } From 89c75dcd5a2e3f1bf0d097cf7c1b36a024e56c32 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 11:56:03 +0000 Subject: [PATCH 09/27] Added in loft insulation method --- recommendations/Costs.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index efa22fd8..6dc2e7aa 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -84,7 +84,7 @@ class Costs: "labour_hours": labour_hours } - def cavity_wall_insulation(self, material): + def loft_insulation(self, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -92,26 +92,21 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 - material = { - "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " - "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " - "retaining discs", - "depth": 75, - "thermal_conductivity": 0.037, - "prime_cost": 5.17, - "material_cost": 5.62, - "labour_cost": 2.25, - "labour_hours": 0.13 - } + # material = { + # "description": "Crown Loft Roll 44 glass fibre roll", + # "depth": 270, + # "thermal_conductivity": 0.044, + # "prime_cost": None, + # "material_cost": 5.91938, + # "labour_cost": 1.96, + # "labour_hours": 0.11 + # } material_cost_per_m2 = material["material_cost"] - wall_area = self.property.insulation_wall_area + floor_area = self.property.floor_area - # This is the amount of material required in m3, assuming a standard 75mm depth - volume = 0.075 * wall_area - - base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area + base_material_cost = material_cost_per_m2 * floor_area + labour_cost = material["labour_cost"] * floor_area subtotal_before_profit = base_material_cost + labour_cost @@ -125,7 +120,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * wall_area + labour_hours = material["labour_hours"] * floor_area return { "total": total_cost, From 4848c2f9c1d72cdc6b45a66fe83f6e1418fd077b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 17:33:55 +0000 Subject: [PATCH 10/27] implemented internal_wall_insulation cost method --- recommendations/Costs.py | 224 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 213 insertions(+), 11 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 6dc2e7aa..b78d8627 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,3 +1,6 @@ +import numpy as np + + class Costs: """ A class to calculate the costs associated with construction works, @@ -38,17 +41,19 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 - material = { - "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or other " - "equal; as full or partial cavity fill; including cutting and fitting around wall ties and " - "retaining discs", - "depth": 75, - "thermal_conductivity": 0.037, - "prime_cost": 5.17, - "material_cost": 5.62, - "labour_cost": 2.25, - "labour_hours": 0.13 - } + # material = { + # "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or + # other " + # "equal; as full or partial cavity fill; including cutting and fitting around wall ties + # and " + # "retaining discs", + # "depth": 75, + # "thermal_conductivity": 0.037, + # "prime_cost": 5.17, + # "material_cost": 5.62, + # "labour_cost": 2.25, + # "labour_hours": 0.13 + # } material_cost_per_m2 = material["material_cost"] wall_area = self.property.insulation_wall_area @@ -132,3 +137,200 @@ class Costs: "profit": profit_cost, "labour_hours": labour_hours } + + def internal_wall_insulation(self, material, non_insulation_materials): + """ + Broadly speaking, the high level steps to an internal wall insulation job are the following: + + 1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles. + It's important to factor in the disposal of debris and the potential need for additional protective + measures to ensure the safety of the work area. + + 2) Insulation Installation: This is the core part of the process where the chosen insulation material is + applied. The choice of insulation material will depend on several factors including thermal performance, + wall construction, and space constraints. + + 3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation, + which can compromise its effectiveness and lead to mold growth. + + 4) Re-decoration: This involves applying plaster to the wall and then painting. + The quality of finish here is important for both aesthetic and functional reasons. + + 5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames, + or window sills might be necessary. + :return: + """ + + # Parsing the provided table into a list of dictionaries + + # non_insulation_materials = [ + # {'type': 'iwi_wall_demolition', + # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, + # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, + # {'type': 'iwi_wall_demolition', + # 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, + # 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, + # {'type': 'iwi_wall_demolition', + # 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' + # 'plaster', + # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, + # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Visqueen High Performance Vapour Barrier', + # 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, + # 'link': 'SPONs', + # 'material_cost': 1.21, + # 'plant_cost': 0, + # 'prime_material_cost': 0.58, + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 1.69, + # 'type': 'iwi_vapour_barrier'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' + # 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', + # 'labour_cost': 6.58, + # 'labour_hours_per_unit': 0.25, + # 'link': "", + # 'material_cost': 0.06, + # 'plant_cost': 0, + # 'prime_material_cost': 0.0, + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 6.64, + # 'type': 'iwi_redecoration'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' + # '5m high', + # 'labour_cost': 0.0, + # 'labour_hours_per_unit': 0.21, + # 'link': "", + # 'material_cost': 0.41, + # 'plant_cost': 0, + # 'prime_material_cost': "", + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 4.34, + # 'type': 'iwi_redecoration'}, + # {'Notes': "", + # 'cost_unit': "", + # 'depth': "", + # 'depth_unit': "", + # 'description': 'Fitting existing softwood skirting or architrave to new ' + # 'frames; 150mm high', + # 'labour_cost': 4.87, + # 'labour_hours_per_unit': 0.01, + # 'link': "", + # 'material_cost': 4.86, + # 'plant_cost': 0, + # 'prime_material_cost': "", + # 'thermal_conductivity': "", + # 'thermal_conductivity_unit': "", + # 'total_cost': 4.88, + # 'type': 'iwi_finishes'} + # ] + # + # material = { + # "type": "internal_wall_insulation", + # "description": "Ecotherm Eco-Versal PIR Insulation Board", + # "depth": 150, + # "depth_unit": "mm", + # "cost_unit": "gbp_per_m2", + # "thermal_conductivity": 0.022, + # "thermal_conductivity_unit": "watt_per_meter_kelvin", + # "prime_material_cost": "", + # "material_cost": 11.68, + # "labour_cost": 3.12, + # "labour_hours_per_unit": 0.18, + # "plant_cost": "", + # "total_cost": 14.8, + # "link": "SPONs" + # } + + # Cost per m2 + wall_area = self.property.insulation_wall_area + + # Extract and check the different types of data we'll need + demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] + finishes_data = [x for x in non_insulation_materials if x["type"] == "iwi_finishes"] + if not demolition_data: + raise ValueError("No data found for iwi_wall_demolition") + + if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2) or (len(finishes_data) != 1): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + # Since we don't know the exact wall construction, we take an average for demolition costs, since + # the cost will depend on the type of wall construction + demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * wall_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area + redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) + finishes_material_costs = finishes_data[0]["material_cost"] * wall_area + + demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) + + demolition_labour_costs = sum([x["labour_cost"] * wall_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * wall_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area + redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) + finishes_labour_costs = finishes_data[0]["labour_cost"] * wall_area + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs + finishes_labour_costs) + + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs + finishes_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * wall_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) + finishes_labour_hours = finishes_data[0]["labour_hours_per_unit"] * wall_area + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours + finishes_labour_hours) + + # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people + labour_days = (labour_hours / 8) / 5 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From 20bc4cab0e527bec5821a0b96f10c4bf3a253959 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 17:59:53 +0000 Subject: [PATCH 11/27] Added wip labour adjustment table --- recommendations/Costs.py | 64 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index b78d8627..fafc15aa 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,5 +1,53 @@ import numpy as np +# Example - delete me +from backend.Property import Property + +print("DELETE ME IN COSTS CLASS") +from epc_api.client import EpcClient + +epc_client = EpcClient(auth_token=AUTH_TOKEN) +p1 = Property( + postcode="NN1 5JY", + address1="2 South Terrace", + epc_client=epc_client, + id=0 +) + +p2 = Property( + postcode="PO12 4TY", + address1="25 Albert Street", + epc_client=epc_client, + id=0 +) + +p1.search_address_epc() +p2.search_address_epc() + +p1.set_basic_property_dimensions() +p2.set_basic_property_dimensions() + +regional_labour_variations = [ + {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, + {"Region": "Inner London", "Adjustment_Factor": 1.05}, + {"Region": "South East", "Adjustment_Factor": 0.96}, + {"Region": "South West", "Adjustment_Factor": 0.90}, + {"Region": "East of England", "Adjustment_Factor": 0.93}, + {"Region": "East Midlands", "Adjustment_Factor": 0.88}, + {"Region": "West Midlands", "Adjustment_Factor": 0.87}, + {"Region": "North East", "Adjustment_Factor": 0.83}, + {"Region": "North West", "Adjustment_Factor": 0.88}, + {"Region": "Yorkshire and Humberside", "Adjustment_Factor": 0.86}, + {"Region": "Wales", "Adjustment_Factor": 0.88}, + {"Region": "Scotland", "Adjustment_Factor": 0.88}, + {"Region": "Northern Ireland", "Adjustment_Factor": 0.76} +] + +county_map = { + "Northamptonshire": "East Midlands", + "Hampshire": "South East", +} + class Costs: """ @@ -32,6 +80,16 @@ class Costs: if not hasattr(property_instance, 'insulation_wall_area'): raise ValueError("Property instance must have an 'insulation_wall_area' attribute") self.property = property_instance + self.regional_labour_variations = regional_labour_variations + + self.county = county_map.get(self.property.data["county"], None) + if self.county is None: + raise ValueError("County not found in county map") + + self.labour_adjustment_factor = [ + x["Adjustment_Factor"] for x in self.regional_labour_variations if + x["Region"] == self.county + ][0] def cavity_wall_insulation(self, material): """ @@ -62,7 +120,7 @@ class Costs: volume = 0.075 * wall_area base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area + labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor subtotal_before_profit = base_material_cost + labour_cost @@ -111,7 +169,7 @@ class Costs: floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area - labour_cost = material["labour_cost"] * floor_area + labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor subtotal_before_profit = base_material_cost + labour_cost @@ -296,6 +354,8 @@ class Costs: labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + redecoration_labour_costs + finishes_labour_costs) + labour_costs = labour_costs * self.labour_adjustment_factor + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + redecoration_material_costs + finishes_material_costs) From 5aebd43c4b34bf04083153036fc463195d6b49f6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 18:02:18 +0000 Subject: [PATCH 12/27] added ValueError for no labour adjustment factor found --- recommendations/Costs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index fafc15aa..c855e690 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -91,6 +91,9 @@ class Costs: x["Region"] == self.county ][0] + if not self.labour_adjustment_factor: + raise ValueError("Labour adjustment factor not found") + def cavity_wall_insulation(self, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, From 9adfa4c07538bd03228cf6cdc991c1c9b5f93b92 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 22 Nov 2023 22:29:55 +0000 Subject: [PATCH 13/27] implemented suspended floor insulation --- backend/Property.py | 5 +- recommendations/Costs.py | 120 ++++++++++++++++++++++++ recommendations/recommendation_utils.py | 19 ---- 3 files changed, 122 insertions(+), 22 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 370eca06..4106c60a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1,5 +1,4 @@ from datetime import datetime -from collections import Counter import re import os import pandas as pd @@ -13,7 +12,7 @@ from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup from recommendations.recommendation_utils import ( - estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area + estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area ) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') @@ -596,7 +595,7 @@ class Property(Definitions): self.number_of_rooms = float(self.data["number-habitable-rooms"]) if self.data["property-type"] == "House": - self.number_of_floors = estimate_floors(self.floor_area, self.number_of_rooms) + self.number_of_floors = 2 elif self.data["property-type"] in ["Flat", "Bungalow"]: self.number_of_floors = 1 elif self.data["property-type"] == "Maisonette": diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c855e690..49298d5f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -27,6 +27,12 @@ p2.search_address_epc() p1.set_basic_property_dimensions() p2.set_basic_property_dimensions() +import pandas as pd + +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - suspended_floor_insulation.csv") +df = df.to_dict("records") + +# This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, {"Region": "Inner London", "Adjustment_Factor": 1.05}, @@ -397,3 +403,117 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def suspended_floor_insulation(self, material, non_insulation_materials): + """ + We characterise the steps for suspended floor insulation as the following tasks: + + 1) Removal of Carpet and Underfelt: Where necessary, remove existing floor coverings to access the floorboards. + 2) Removal of Floor Boarding: Carefully remove floorboards to access the space beneath for insulation. + 3) Installation of Vapour Barrier: Install a vapour barrier to prevent moisture from affecting + the insulation and floor structure. + 4) Installation of Insulation: Fit the chosen insulation material between the joists in the floor void. + 5) Refixing Floorboards: Replace and secure the floorboards after insulation installation. + 6) Re-carpeting: Lay down the carpet or other floor coverings once the insulation and floorboards are in place. + :return: + """ + + # material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0, + # 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + # 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0, + # 'total_cost': 13.46, 'link': 'SPONs', + # 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. We use + # the ' + # 'same values as in Crown loft roll 44, since it is also an insulation roll'} + # + # non_insulation_materials = [ + # {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + # 'therefore there is no need for a skip'}, + # {'type': 'suspended_floor_demolition', + # 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, + # 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, + # 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + # 'therefore we need just labour rates'}] + + insulation_floor_area = self.property.floor_area / self.property.number_of_floors + + demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"] + + if (len(demolition_data) != 2) or (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + demolition_material_costs = sum([x["material_cost"] * insulation_floor_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * insulation_floor_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area + redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data]) + + demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * insulation_floor_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area + redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data]) + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * self.VAT_RATE + + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data]) + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours) + + # Assume a team of 3 people for a small to medium size project + labour_days = (labour_hours / 8) / 3 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 063a274c..7cba8257 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -527,25 +527,6 @@ def get_wall_type( return None -def estimate_floors(floor_area, num_rooms): - """ - Simple utility funciton, which assuming a 15m squared room, estimates the number of floors in a property - :param floor_area: Gross floor area of a property - :param num_rooms: Number of rooms in a property - :return: Number of floors in a property - """ - # Estimate total room area - total_room_area = num_rooms * 15 - - # Estimate the number of floors - floors = floor_area / total_room_area - - # Round up to the nearest whole number - floors = round(floors) - - return floors - - def estimate_wall_area(num_floors, floor_height, perimeter): wall_area_one_floor = perimeter * floor_height From dea9a7fb17f28f21bb545666aa68a5bde36184ac Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 08:47:37 +0000 Subject: [PATCH 14/27] implemented solid floor insulation --- recommendations/Costs.py | 205 +++++++++++++++++++++++++++++++++++---- 1 file changed, 187 insertions(+), 18 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 49298d5f..1c8cf6d8 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -29,7 +29,7 @@ p2.set_basic_property_dimensions() import pandas as pd -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - suspended_floor_insulation.csv") +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - solid_floor_insulation.csv") df = df.to_dict("records") # This data comes from SPONs @@ -69,6 +69,9 @@ class Costs: # We assume a conservative 10% contingency for all works which is a rate defined by SPONs CONTINGENCY = 0.1 + # Where there is more uncertainty, a higher contingency rate is used + HIGH_RISH_CONTINGENCY = 0.15 + # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% # rate, on the total cost before VAT, as recommended by SPONs @@ -100,7 +103,7 @@ class Costs: if not self.labour_adjustment_factor: raise ValueError("Labour adjustment factor not found") - def cavity_wall_insulation(self, material): + def cavity_wall_insulation(self, wall_area, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -123,7 +126,7 @@ class Costs: # } material_cost_per_m2 = material["material_cost"] - wall_area = self.property.insulation_wall_area + # wall_area = self.property.insulation_wall_area # This is the amount of material required in m3, assuming a standard 75mm depth volume = 0.075 * wall_area @@ -156,7 +159,7 @@ class Costs: "labour_hours": labour_hours } - def loft_insulation(self, material): + def loft_insulation(self, floor_area, material): """ Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. @@ -175,7 +178,7 @@ class Costs: # } material_cost_per_m2 = material["material_cost"] - floor_area = self.property.floor_area + # floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor @@ -205,7 +208,7 @@ class Costs: "labour_hours": labour_hours } - def internal_wall_insulation(self, material, non_insulation_materials): + def internal_wall_insulation(self, wall_area, material, non_insulation_materials): """ Broadly speaking, the high level steps to an internal wall insulation job are the following: @@ -309,7 +312,7 @@ class Costs: # 'thermal_conductivity': "", # 'thermal_conductivity_unit': "", # 'total_cost': 4.88, - # 'type': 'iwi_finishes'} + # 'type': 'iwi_redecoration'} # ] # # material = { @@ -330,17 +333,16 @@ class Costs: # } # Cost per m2 - wall_area = self.property.insulation_wall_area + # wall_area = self.property.insulation_wall_area # Extract and check the different types of data we'll need demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] - finishes_data = [x for x in non_insulation_materials if x["type"] == "iwi_finishes"] if not demolition_data: raise ValueError("No data found for iwi_wall_demolition") - if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2) or (len(finishes_data) != 1): + if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3): raise ValueError("Incorrect number of data entries for non-insulation materials") # Break out the individual material costs @@ -350,7 +352,6 @@ class Costs: insulation_material_costs = material["material_cost"] * wall_area vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) - finishes_material_costs = finishes_data[0]["material_cost"] * wall_area demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) @@ -358,15 +359,14 @@ class Costs: insulation_labour_costs = material["labour_cost"] * wall_area vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) - finishes_labour_costs = finishes_data[0]["labour_cost"] * wall_area labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + - redecoration_labour_costs + finishes_labour_costs) + redecoration_labour_costs) labour_costs = labour_costs * self.labour_adjustment_factor materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + - redecoration_material_costs + finishes_material_costs) + redecoration_material_costs) subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs @@ -384,10 +384,9 @@ class Costs: insulation_labour_hours = material["labour_hours_per_unit"] * wall_area vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) - finishes_labour_hours = finishes_data[0]["labour_hours_per_unit"] * wall_area labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + - redecoration_labour_hours + finishes_labour_hours) + redecoration_labour_hours) # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people labour_days = (labour_hours / 8) / 5 @@ -404,7 +403,7 @@ class Costs: "labour_days": labour_days } - def suspended_floor_insulation(self, material, non_insulation_materials): + def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): """ We characterise the steps for suspended floor insulation as the following tasks: @@ -455,7 +454,7 @@ class Costs: # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' # 'therefore we need just labour rates'}] - insulation_floor_area = self.property.floor_area / self.property.number_of_floors + # insulation_floor_area = self.property.floor_area / self.property.number_of_floors demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] @@ -517,3 +516,173 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): + """ + We characterise the steps for solid floor insulation as the following tasks: + + 1) Removal of Carpet and Underfelt: This is the initial stage where any existing floor coverings, like carpets, + tiles, or linoleum, are carefully removed. This exposes the solid floor beneath, which is typically concrete. + + 2) Preparation of Flooring: This step is critical. It involves: + - Cleaning the existing floor surface thoroughly to remove debris and ensure a flat surface. + - Assessing and repairing any damage to the concrete floor. This might include filling cracks or leveling + uneven areas. + + 3) Installation of a Damp Proof Membrane (DPM): Before installing insulation, a DPM is often laid down to + prevent moisture from rising into the insulation and the interior space. This step is crucial in areas prone to + dampness. + + 4) Install Insulation: The insulation, often in the form of rigid foam boards, is laid over the DPM. + The choice of insulation material will depend on the desired thermal properties and the available floor height. + Care is taken to minimize thermal bridges and ensure a snug fit between insulation boards. + + 5) Laying a New Subfloor: Over the insulation, a new subfloor is often installed. This could be a layer of + screed (a type of concrete) or wooden boarding, depending on the specific requirements and preferences. + + 6) Re-decoration and Finishing Touches: Once the subfloor is in place and has set or dried (if necessary), + the final floor finish can be applied. This might involve: + - Laying new tiles, wooden flooring, or other chosen materials. + - If you're planning to re-carpet, this would be the stage to do it. + - Skirting boards may need to be refitted or replaced. + + 7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door + thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments. + :param material: + :param non_insulation_materials: + :return: + """ + + material = { + 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + } + + non_insulation_materials = [ + {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'solid_floor_preparation', + 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + 'description': 'Clean out crack to ' + 'form a 20mm×20mm ' + 'groove and fill with ' + 'cement: mortar mixed ' + 'with bonding agent', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, + 'material_cost': 6.91, + 'labour_cost': 18.99, + 'labour_hours_per_unit': 0.61, + 'plant_cost': 0.16, + 'total_cost': 26.06, 'link': 0, + 'Notes': 'This step is the ' + 'assessment and repair of ' + 'any damage to the concrete ' + 'floor such as filling ' + 'cracks or levelling uneven ' + 'areas'}, + {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'solid_floor_redecoration', + 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + 'Notes': 'This is the screed layer, placed on top of the insulation'}, + {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}, + {'type': 'solid_floor_redecoration', + 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + ] + + # insulation_floor_area = self.property.floor_area / self.property.number_of_floors + + demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"] + preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"] + vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_redecoration"] + + if ((len(demolition_data) != 1) or (len(preparation_data) != 2) or (len(vapour_barrier_data) != 1) or + (len(redecoration_data) != 3)): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + preparation_material_costs = sum([x["material_cost"] * insulation_floor_area for x in preparation_data]) + insulation_material_costs = material["material_cost"] * insulation_floor_area + vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area + redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data]) + + demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data]) + preparation_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in preparation_data]) + insulation_labour_costs = material["labour_cost"] * insulation_floor_area + vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area + redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data]) + + preparation_plant_costs = sum([x["plant_cost"] * insulation_floor_area for x in preparation_data]) + + labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + + redecoration_labour_costs + preparation_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_cost = (preparation_material_costs + insulation_material_costs + vapour_barrier_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_cost + preparation_plant_costs + + # We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors, + # as well as scope for damage to the existing floor during preparation. + contingency_cost = subtotal_before_profit * self.HIGH_RISH_CONTINGENCY + preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + vat_cost = subtotal_before_vat * self.VAT_RATE + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data]) + preparation_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in preparation_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area + vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area + redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data]) + + labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + + redecoration_labour_hours + preparation_labour_hours) + + # Assume a team of 3 people for a small to medium size project + labour_days = (labour_hours / 8) / 3 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_cost, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From 63de7c19df65dfdc58f51dd3a62230ca95cf8129 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 14:51:30 +0000 Subject: [PATCH 15/27] completed pricing for ewi --- recommendations/Costs.py | 348 +++++++++++++++++++++++++++++++-------- 1 file changed, 278 insertions(+), 70 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 1c8cf6d8..c6e4ceb7 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -29,7 +29,7 @@ p2.set_basic_property_dimensions() import pandas as pd -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - solid_floor_insulation.csv") +df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - external_wall_insulation.csv") df = df.to_dict("records") # This data comes from SPONs @@ -61,8 +61,11 @@ class Costs: specifically focusing on cavity wall insulation. It includes contingency, preliminaries, profit margin, and VAT calculations. - As a sense check, there is a useful article from checkatrade on retrofitting walls and expected costs: + As a sense check, there is a useful article from checkatrade on retrofitting and expected costs: https://www.checkatrade.com/blog/cost-guides/retrofit-insulation-cost/ + + Another useful article for benchmarking the cost of floor insulation: + https://www.checkatrade.com/blog/cost-guides/floor-insulation-cost/ """ # Contingency is a percentage of the total cost of the work and covers unforseen expenses @@ -70,13 +73,22 @@ class Costs: CONTINGENCY = 0.1 # Where there is more uncertainty, a higher contingency rate is used - HIGH_RISH_CONTINGENCY = 0.15 + HIGH_RISK_CONTINGENCY = 0.15 + # When there is less uncertainty, a lower contingency rate is used + LOW_RISK_CONTINGENCY = 0.05 # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10% # rate, on the total cost before VAT, as recommended by SPONs PRELIMINARIES = 0.1 + # For higher risk projects, a higher preliminaries rate is used. SPONs indicates that a higher risk project might + # have a preliminaries of 12-14% so we use 12% as the median for the preliminaries rate. + # For External wall insulation (EWI), we use 15% as the preliminaries rate if we think the property might + # need scaffolding, otherwise we use 12%. This is to account for any site preparation that might be required + EWI_NO_SCAFFOLDING_PRELIMINARIES = 0.12 + EWI_SCAFFOLDING_PRELIMINARIES = 0.15 + VAT_RATE = 0.2 PROFIT_MARGIN = 0.15 @@ -355,7 +367,8 @@ class Costs: demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - demolition_labour_costs = sum([x["labour_cost"] * wall_area for x in demolition_data]) + # Again for demolition, we average since we aren't sure which demolition process will be used + demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) insulation_labour_costs = material["labour_cost"] * wall_area vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) @@ -380,7 +393,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - demolition_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) insulation_labour_hours = material["labour_hours_per_unit"] * wall_area vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) @@ -553,70 +566,70 @@ class Costs: :return: """ - material = { - 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, - 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 - } - - non_insulation_materials = [ - {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - 'therefore there is no need for a skip'}, - {'type': 'solid_floor_preparation', - 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', - 'description': 'Clean out crack to ' - 'form a 20mm×20mm ' - 'groove and fill with ' - 'cement: mortar mixed ' - 'with bonding agent', - 'depth': 0, 'depth_unit': 0, - 'cost_unit': 0, - 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, - 'material_cost': 6.91, - 'labour_cost': 18.99, - 'labour_hours_per_unit': 0.61, - 'plant_cost': 0.16, - 'total_cost': 26.06, 'link': 0, - 'Notes': 'This step is the ' - 'assessment and repair of ' - 'any damage to the concrete ' - 'floor such as filling ' - 'cracks or levelling uneven ' - 'areas'}, - {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - {'type': 'solid_floor_redecoration', - 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, - 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', - 'Notes': 'This is the screed layer, placed on top of the insulation'}, - {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - 'therefore we need just labour rates'}, - {'type': 'solid_floor_redecoration', - 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, - 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} - ] + # material = { + # 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + # 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + # 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + # 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + # } + # + # non_insulation_materials = [ + # {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + # 'therefore there is no need for a skip'}, + # {'type': 'solid_floor_preparation', + # 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + # 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + # 'description': 'Clean out crack to ' + # 'form a 20mm×20mm ' + # 'groove and fill with ' + # 'cement: mortar mixed ' + # 'with bonding agent', + # 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, + # 'material_cost': 6.91, + # 'labour_cost': 18.99, + # 'labour_hours_per_unit': 0.61, + # 'plant_cost': 0.16, + # 'total_cost': 26.06, 'link': 0, + # 'Notes': 'This step is the ' + # 'assessment and repair of ' + # 'any damage to the concrete ' + # 'floor such as filling ' + # 'cracks or levelling uneven ' + # 'areas'}, + # {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + # {'type': 'solid_floor_redecoration', + # 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + # 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + # 'Notes': 'This is the screed layer, placed on top of the insulation'}, + # {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + # 'therefore we need just labour rates'}, + # {'type': 'solid_floor_redecoration', + # 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + # 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + # ] # insulation_floor_area = self.property.floor_area / self.property.number_of_floors @@ -655,7 +668,7 @@ class Costs: # We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors, # as well as scope for damage to the existing floor during preparation. - contingency_cost = subtotal_before_profit * self.HIGH_RISH_CONTINGENCY + contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES profit_cost = subtotal_before_profit * self.PROFIT_MARGIN @@ -686,3 +699,198 @@ class Costs: "labour_hours": labour_hours, "labour_days": labour_days } + + def external_wall_insulation(self, wall_area, material, non_insulation_materials): + """ + We characterise external wall insulation as the following steps: + + 1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective + sheets to protect the flooring and landscaping around the work area. + + 2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or + detached houses. For terraced houses or lower-level work, scaffolding might not be necessary. + + 3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material, + and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement + of the top surface may be necessary. + + 4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation + boards and stabilize the wall surface, especially if it's old or weathered. + + 5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings, + or a combination of both. + + 6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation. + This layer provides strength and helps prevent cracking. + + 7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation + and provides an aesthetic look. + + 8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that + were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness. + + 9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system, + followed by cleaning up the site to remove all debris and materials. + + In the actual materials data, at this point, we have costing for: + - wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition) + - wall surface cleaning and priming (ewi_wall_preparation) + - insulation (external_wall_insulation) + - basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh) + + All of this data comes from SPONS, however there are some clear features missing. Because we could not find + suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding, + re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by + increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these + steps should be included in the materials data. We will look to improve this in the future, with data from + installers + + :param wall_area: + :param material: + :param non_insulation_materials: + :return: + """ + + # For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding + + if self.property.data["property-type"] == "House": + if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]: + preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES + else: + preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES + elif self.property.data["property-type"] == "Maisonette": + preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES + elif self.property.data["property-type"] == "Bungalow": + preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES + else: + raise ValueError("Unsupported property type - haven't handled flats") + + # non_insulation_materials = [x for x in df if x["type"] != "external_wall_insulation"] + # insulation_materials = [x for x in df if x["type"] == "external_wall_insulation"] + # material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', + # 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, + # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, + # 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, + # 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} + # non_insulation_materials = [ + # {'type': 'ewi_wall_demolition', + # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' + # 'hammer; plaster to walls.', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, + # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, + # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', + # 'description': 'Stud walls: Remove wall linings ' + # 'including battening behind; ' + # 'plasterboard and skim', + # 'depth': 0, 'depth_unit': 0, + # 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, + # 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, + # 'plant_cost': 1.25, 'total_cost': 7.48, + # 'link': 'SPONs', 'Notes': 0}, + # {'type': 'ewi_wall_demolition', + # 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' + # 'behind; wood lath and plaster', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, + # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, + # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', + # 'description': 'Clean and prepare surfaces, ' + # 'one coat Keim dilution, ' + # 'one coat primer and two coats ' + # 'of Keim Ecosil paint; Brick or ' + # 'block walls; over 300 mm girth', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, + # 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, + # 'prime_material_cost': 0, 'material_cost': 7.3, + # 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, + # 'plant_cost': 0, 'total_cost': 12.92, + # 'link': 'SPONs', + # 'Notes': 'This work covers the preparation and ' + # 'priming of the wall before insulating'}, + # {'type': 'ewi_wall_redecoration', + # 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' + # 'separately) with horizontal PVC intermediate track and vertical ' + # 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' + # 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' + # 'Topcoat Render (white)', + # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, + # 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, + # 'total_cost': 69.94, 'link': 'SPONs', + # 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' + # 'cost of 99.17 per meter square. This includes the cost of insulation. ' + # 'To get the costing for just the works and not the insulation, ' + # 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' + # 'as an example, which costs £29.23 per meter square, giving us the cost ' + # 'of the remaining works without insulation. This material gives us a ' + # 'cost for basecoat, mesh application and a render finish'}] + + demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] + preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] + redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] + + if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1): + raise ValueError("Incorrect number of data entries for non-insulation materials") + + # Break out the individual material costs + # Since we don't know the exact wall construction, we take an average for demolition costs, since + # the cost will depend on the type of wall construction + demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) + insulation_material_costs = material["material_cost"] * wall_area + preparation_material_costs = preparation_data[0]["material_cost"] * wall_area + redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area + + demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) + + demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) + insulation_labour_costs = material["labour_cost"] * wall_area + preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area + redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area + + labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs + + preparation_labour_costs) + + labour_costs = labour_costs * self.labour_adjustment_factor + + materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs + + redecoration_material_costs) + + subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs + + contingency_cost = subtotal_before_profit * self.CONTINGENCY + preliminaries_cost = subtotal_before_profit * preliminaries_rate + profit_cost = subtotal_before_profit * self.PROFIT_MARGIN + + subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost + vat_cost = subtotal_before_vat * self.VAT_RATE + total_cost = subtotal_before_vat + vat_cost + + demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) + insulation_labour_hours = material["labour_hours_per_unit"] * wall_area + preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area + redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area + + labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours + + preparation_labour_hours) + + # Assume a team of 3-5 people for a small to medium size project + labour_days = (labour_hours / 8) / 4 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": materials_costs, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_days": labour_days + } From a9a7d3f3a809fe40281cf732a26e2359fea9ae44 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:05:04 +0000 Subject: [PATCH 16/27] Added test_external_wall_area --- backend/Property.py | 9 ++++--- recommendations/Costs.py | 24 +++++++------------ recommendations/recommendation_utils.py | 23 ++++++++++++++++-- recommendations/tests/test_costs.py | 0 .../tests/test_recommendation_utils.py | 15 ++++++++++++ 5 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 recommendations/tests/test_costs.py diff --git a/backend/Property.py b/backend/Property.py index 4106c60a..8bdfa7e9 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -12,7 +12,7 @@ from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup from recommendations.recommendation_utils import ( - estimate_perimeter, get_wall_type, estimate_wall_area, esimtate_pitched_roof_area + estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area ) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') @@ -612,8 +612,11 @@ class Property(Definitions): self.floor_area / self.number_of_floors, self.number_of_rooms / self.number_of_floors ) - self.insulation_wall_area = estimate_wall_area( - num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter + self.insulation_wall_area = estimate_external_wall_area( + num_floors=self.number_of_floors, + floor_height=self.floor_height, + perimeter=self.perimeter, + built_form=self.data["built-form"], ) self.pitched_roof_area = esimtate_pitched_roof_area( diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c6e4ceb7..d14b5f33 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -27,11 +27,6 @@ p2.search_address_epc() p1.set_basic_property_dimensions() p2.set_basic_property_dimensions() -import pandas as pd - -df = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/Hestia Materials - external_wall_insulation.csv") -df = df.to_dict("records") - # This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, @@ -120,28 +115,25 @@ class Costs: Calculates the total cost for cavity wall insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. + Because of some limitations in the SPONs data, there are no materials that can be blown through a wall, + therefore we have adapted similar materials, basing our estimates on 75mm cavity slabs, and have halved the + labour time required. That is why we still price based on wall area despite volume actually being the correct + metric. + :return: A dictionary containing detailed cost breakdown. """ # Cost per m2 # material = { - # "description": "Crown Dritherm Cavity Slab 37 (Thermal conductivity 0.037 W/mK) glass fibre batt or - # other " - # "equal; as full or partial cavity fill; including cutting and fitting around wall ties - # and " - # "retaining discs", + # "description": "cwi", # "depth": 75, # "thermal_conductivity": 0.037, # "prime_cost": 5.17, # "material_cost": 5.62, - # "labour_cost": 2.25, - # "labour_hours": 0.13 + # "labour_cost": 1.125, + # "labour_hours": 0.065 # } material_cost_per_m2 = material["material_cost"] - # wall_area = self.property.insulation_wall_area - - # This is the amount of material required in m3, assuming a standard 75mm depth - volume = 0.075 * wall_area base_material_cost = material_cost_per_m2 * wall_area labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 7cba8257..217f313f 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -527,12 +527,31 @@ def get_wall_type( return None -def estimate_wall_area(num_floors, floor_height, perimeter): +def estimate_external_wall_area(num_floors, floor_height, perimeter, built_form): + """ + This method estimates the external wall area based on fundamental assumptions about the home + + + :param num_floors: Number of floors in the building. + :param floor_height: Height of one floor in meters. + :param perimeter: Total perimeter of the building on one floor in meters. + :param built_form: The built form of the property. This is used to determine the number of exposed walls. + :return: + """ wall_area_one_floor = perimeter * floor_height total_wall_area = wall_area_one_floor * num_floors - return total_wall_area + number_exposed_walls = { + 'End-Terrace': 3, + 'Mid-Terrace': 2, + 'Semi-Detached': 3, + 'Detached': 4, + } + + exposed_wall_area = total_wall_area * (number_exposed_walls[built_form] / 4) + + return exposed_wall_area def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py new file mode 100644 index 00000000..e69de29b diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 22280ed5..73796979 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -405,3 +405,18 @@ def test_esimtate_pitched_roof_area(): ) assert zero_roof_area2 == 0 + + +def test_external_wall_area(): + # Arrange: Define the test cases + test_cases = [ + (2, 3, 40, 'End-Terrace', 180), # 3 exposed walls + (2, 3, 40, 'Mid-Terrace', 120), # 2 exposed walls + (2, 3, 40, 'Semi-Detached', 180), # 3 exposed walls + (2, 3, 40, 'Detached', 240), # 4 exposed walls + ] + + # Act and Assert: Run the test cases + for num_floors, floor_height, perimeter, built_form, expected in test_cases: + result = recommendation_utils.estimate_external_wall_area(num_floors, floor_height, perimeter, built_form) + assert result == expected, f"Test failed for {built_form}: Expected {expected}, got {result}" From 824b92fedd908bb4dd78368f431bb9681816fb0e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:10:07 +0000 Subject: [PATCH 17/27] Added cavity wall test --- recommendations/Costs.py | 10 --------- recommendations/tests/test_costs.py | 33 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index d14b5f33..1fb1114d 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -122,16 +122,6 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # Cost per m2 - # material = { - # "description": "cwi", - # "depth": 75, - # "thermal_conductivity": 0.037, - # "prime_cost": 5.17, - # "material_cost": 5.62, - # "labour_cost": 1.125, - # "labour_hours": 0.065 - # } material_cost_per_m2 = material["material_cost"] diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index e69de29b..8ca12e0b 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -0,0 +1,33 @@ +from recommendations.Costs import Costs +from unittest.mock import Mock + + +class TestCosts: + + def test_cavity_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + + cwi_material = { + "description": "cwi", + "depth": 75, + "thermal_conductivity": 0.037, + "prime_cost": 5.17, + "material_cost": 5.62, + "labour_cost": 1.125, + "labour_hours": 0.065 + } + + cwi_results = costs.cavity_wall_insulation( + wall_area=95.9104281347967, + material=cwi_material, + ) + + assert cwi_results == {'total': 1027.0280465530302, 'subtotal': 855.8567054608585, 'vat': 171.1713410921717, + 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, + 'material': 539.0166061175574, 'profit': 95.09518949565093, + 'labour_hours': 6.234177828761786} From 11ba4c64d08efe5fbae89c3910fb79721ed59192 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:19:22 +0000 Subject: [PATCH 18/27] tested iwi costs --- backend/Property.py | 5 +- recommendations/Costs.py | 121 +------------------- recommendations/FloorRecommendations.py | 4 +- recommendations/RoofRecommendations.py | 1 + recommendations/tests/test_costs.py | 143 ++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 121 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 8bdfa7e9..a3328156 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -82,6 +82,7 @@ class Property(Definitions): self.insulation_wall_area = None self.floor_area = None self.pitched_roof_area = None + self.insulation_floor_area = None if epc_client: self.epc_client = epc_client @@ -619,8 +620,10 @@ class Property(Definitions): built_form=self.data["built-form"], ) + self.insulation_floor_area = self.floor_area / self.number_of_floors + self.pitched_roof_area = esimtate_pitched_roof_area( - floor_area=self.floor_area / self.number_of_floors, floor_height=self.floor_height + floor_area=self.insulation_floor_area, floor_height=self.floor_height ) def set_wall_type(self): diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 1fb1114d..e8f9d122 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -160,19 +160,7 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # Cost per m2 - # material = { - # "description": "Crown Loft Roll 44 glass fibre roll", - # "depth": 270, - # "thermal_conductivity": 0.044, - # "prime_cost": None, - # "material_cost": 5.91938, - # "labour_cost": 1.96, - # "labour_hours": 0.11 - # } - material_cost_per_m2 = material["material_cost"] - # floor_area = self.property.floor_area base_material_cost = material_cost_per_m2 * floor_area labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor @@ -225,110 +213,6 @@ class Costs: :return: """ - # Parsing the provided table into a list of dictionaries - - # non_insulation_materials = [ - # {'type': 'iwi_wall_demolition', - # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, - # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, - # {'type': 'iwi_wall_demolition', - # 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, - # 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, - # {'type': 'iwi_wall_demolition', - # 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' - # 'plaster', - # 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - # 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, - # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Visqueen High Performance Vapour Barrier', - # 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, - # 'link': 'SPONs', - # 'material_cost': 1.21, - # 'plant_cost': 0, - # 'prime_material_cost': 0.58, - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 1.69, - # 'type': 'iwi_vapour_barrier'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' - # 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', - # 'labour_cost': 6.58, - # 'labour_hours_per_unit': 0.25, - # 'link': "", - # 'material_cost': 0.06, - # 'plant_cost': 0, - # 'prime_material_cost': 0.0, - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 6.64, - # 'type': 'iwi_redecoration'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' - # '5m high', - # 'labour_cost': 0.0, - # 'labour_hours_per_unit': 0.21, - # 'link': "", - # 'material_cost': 0.41, - # 'plant_cost': 0, - # 'prime_material_cost': "", - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 4.34, - # 'type': 'iwi_redecoration'}, - # {'Notes': "", - # 'cost_unit': "", - # 'depth': "", - # 'depth_unit': "", - # 'description': 'Fitting existing softwood skirting or architrave to new ' - # 'frames; 150mm high', - # 'labour_cost': 4.87, - # 'labour_hours_per_unit': 0.01, - # 'link': "", - # 'material_cost': 4.86, - # 'plant_cost': 0, - # 'prime_material_cost': "", - # 'thermal_conductivity': "", - # 'thermal_conductivity_unit': "", - # 'total_cost': 4.88, - # 'type': 'iwi_redecoration'} - # ] - # - # material = { - # "type": "internal_wall_insulation", - # "description": "Ecotherm Eco-Versal PIR Insulation Board", - # "depth": 150, - # "depth_unit": "mm", - # "cost_unit": "gbp_per_m2", - # "thermal_conductivity": 0.022, - # "thermal_conductivity_unit": "watt_per_meter_kelvin", - # "prime_material_cost": "", - # "material_cost": 11.68, - # "labour_cost": 3.12, - # "labour_hours_per_unit": 0.18, - # "plant_cost": "", - # "total_cost": 14.8, - # "link": "SPONs" - # } - - # Cost per m2 - # wall_area = self.property.insulation_wall_area - # Extract and check the different types of data we'll need demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] @@ -365,7 +249,8 @@ class Costs: subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - contingency_cost = subtotal_before_profit * self.CONTINGENCY + # We use high risk contingency for iwi + contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES profit_cost = subtotal_before_profit * self.PROFIT_MARGIN @@ -384,7 +269,7 @@ class Costs: redecoration_labour_hours) # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people - labour_days = (labour_hours / 8) / 5 + labour_days = (labour_hours / 8) / 4 return { "total": total_cost, diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index bc24b6c3..5b194e0d 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -58,7 +58,7 @@ class FloorRecommendations(Definitions): ) property_type = self.property.data["property-type"] - floor_area = self.property.floor_area / self.property.number_of_floors + floor_area = self.property.insulation_floor_area year_built = self.property.year_built if self.property.floor["another_property_below"] | (self.property.floor["insulation_thickness"] in [ @@ -137,7 +137,7 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - quantity = self.property.floor_area / self.property.number_of_floors + quantity = self.property.insulation_floor_area estimated_cost = cost_per_unit * quantity diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index bfa63908..4f96f629 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -161,6 +161,7 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + # TODO: We should use the floor area divided by the number of floors to get the area of the roof estimated_cost = cost_per_unit * self.property.floor_area if roof["is_pitched"]: diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 8ca12e0b..8e2d76ce 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -31,3 +31,146 @@ class TestCosts: 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574, 'profit': 95.09518949565093, 'labour_hours': 6.234177828761786} + + def test_loft_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + loft_material = { + "description": "Crown Loft Roll 44 glass fibre roll", + "depth": 270, + "thermal_conductivity": 0.044, + "prime_cost": None, + "material_cost": 5.91938, + "labour_cost": 1.96, + "labour_hours": 0.11 + } + + loft_results = costs.loft_insulation( + floor_area=33.5, + material=loft_material, + ) + + assert loft_results == {'total': 414.8496486, 'subtotal': 345.70804050000004, 'vat': 69.14160810000001, + 'contingency': 25.608003000000004, 'preliminaries': 25.608003000000004, + 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685} + + def test_internal_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + iwi_non_insulation_materials = [ + {'type': 'iwi_wall_demolition', + 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, + 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, + {'type': 'iwi_wall_demolition', + 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, + 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, + {'type': 'iwi_wall_demolition', + 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' + 'plaster', + 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, + 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, + 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Visqueen High Performance Vapour Barrier', + 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, + 'link': 'SPONs', + 'material_cost': 1.21, + 'plant_cost': 0, + 'prime_material_cost': 0.58, + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 1.69, + 'type': 'iwi_vapour_barrier'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' + 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', + 'labour_cost': 6.58, + 'labour_hours_per_unit': 0.25, + 'link': "", + 'material_cost': 0.06, + 'plant_cost': 0, + 'prime_material_cost': 0.0, + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 6.64, + 'type': 'iwi_redecoration'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' + '5m high', + 'labour_cost': 0.0, + 'labour_hours_per_unit': 0.21, + 'link': "", + 'material_cost': 0.41, + 'plant_cost': 0, + 'prime_material_cost': "", + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 4.34, + 'type': 'iwi_redecoration'}, + {'Notes': "", + 'cost_unit': "", + 'depth': "", + 'depth_unit': "", + 'description': 'Fitting existing softwood skirting or architrave to new ' + 'frames; 150mm high', + 'labour_cost': 4.87, + 'labour_hours_per_unit': 0.01, + 'link': "", + 'material_cost': 4.86, + 'plant_cost': 0, + 'prime_material_cost': "", + 'thermal_conductivity': "", + 'thermal_conductivity_unit': "", + 'total_cost': 4.88, + 'type': 'iwi_redecoration'} + ] + + iwi_material = { + "type": "internal_wall_insulation", + "description": "Ecotherm Eco-Versal PIR Insulation Board", + "depth": 150, + "depth_unit": "mm", + "cost_unit": "gbp_per_m2", + "thermal_conductivity": 0.022, + "thermal_conductivity_unit": "watt_per_meter_kelvin", + "prime_material_cost": "", + "material_cost": 11.68, + "labour_cost": 3.12, + "labour_hours_per_unit": 0.18, + "plant_cost": "", + "total_cost": 14.8, + "link": "SPONs" + } + + iwi_results = costs.internal_wall_insulation( + wall_area=95.9104281347967, + material=iwi_material, + non_insulation_materials=iwi_non_insulation_materials + ) + + assert iwi_results == {'total': 6421.5484411659245, 'subtotal': 5351.29036763827, 'vat': 1070.258073527654, + 'contingency': 573.3525393898148, 'preliminaries': 382.2350262598765, + 'material': 1747.488000615996, 'profit': 573.3525393898148, + 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405} From ee698c40b5e8c562a21d6b3bf46def9262bf3833 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 16:30:38 +0000 Subject: [PATCH 19/27] Added unit tests for costs --- recommendations/Costs.py | 197 ++--------------------- recommendations/tests/test_costs.py | 241 +++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 184 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index e8f9d122..8f54a79a 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -150,7 +150,8 @@ class Costs: "preliminaries": preliminaries_cost, "material": base_material_cost, "profit": profit_cost, - "labour_hours": labour_hours + "labour_hours": labour_hours, + "labour_cost": labour_cost } def loft_insulation(self, floor_area, material): @@ -187,7 +188,8 @@ class Costs: "preliminaries": preliminaries_cost, "material": base_material_cost, "profit": profit_cost, - "labour_hours": labour_hours + "labour_hours": labour_hours, + "labour_cost": labour_cost } def internal_wall_insulation(self, wall_area, material, non_insulation_materials): @@ -280,7 +282,8 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -297,45 +300,6 @@ class Costs: :return: """ - # material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0, - # 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - # 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0, - # 'total_cost': 13.46, 'link': 'SPONs', - # 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. We use - # the ' - # 'same values as in Crown loft roll 44, since it is also an insulation roll'} - # - # non_insulation_materials = [ - # {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - # 'therefore there is no need for a skip'}, - # {'type': 'suspended_floor_demolition', - # 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, - # 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, - # 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - # 'therefore we need just labour rates'}] - - # insulation_floor_area = self.property.floor_area / self.property.number_of_floors - demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"] @@ -394,7 +358,8 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -428,78 +393,13 @@ class Costs: 7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments. - :param material: - :param non_insulation_materials: + + :param insulation_floor_area: Area of the floor to be insulated + :param material: Selected insulation material + :param non_insulation_materials: Non-insulation materials required for the job :return: """ - # material = { - # 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - # 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, - # 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, - # 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 - # } - # - # non_insulation_materials = [ - # {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - # 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', - # 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' - # 'therefore there is no need for a skip'}, - # {'type': 'solid_floor_preparation', - # 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - # 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', - # 'description': 'Clean out crack to ' - # 'form a 20mm×20mm ' - # 'groove and fill with ' - # 'cement: mortar mixed ' - # 'with bonding agent', - # 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, - # 'material_cost': 6.91, - # 'labour_cost': 18.99, - # 'labour_hours_per_unit': 0.61, - # 'plant_cost': 0.16, - # 'total_cost': 26.06, 'link': 0, - # 'Notes': 'This step is the ' - # 'assessment and repair of ' - # 'any damage to the concrete ' - # 'floor such as filling ' - # 'cracks or levelling uneven ' - # 'areas'}, - # {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, - # 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, - # {'type': 'solid_floor_redecoration', - # 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, - # 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', - # 'Notes': 'This is the screed layer, placed on top of the insulation'}, - # {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - # 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', - # 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' - # 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' - # 'therefore we need just labour rates'}, - # {'type': 'solid_floor_redecoration', - # 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, - # 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - # 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} - # ] - - # insulation_floor_area = self.property.floor_area / self.property.number_of_floors - demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"] preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"] vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"] @@ -564,7 +464,8 @@ class Costs: "material": materials_cost, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } def external_wall_insulation(self, wall_area, material, non_insulation_materials): @@ -632,73 +533,6 @@ class Costs: else: raise ValueError("Unsupported property type - haven't handled flats") - # non_insulation_materials = [x for x in df if x["type"] != "external_wall_insulation"] - # insulation_materials = [x for x in df if x["type"] == "external_wall_insulation"] - # material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - # 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, - # 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, - # 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, - # 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} - # non_insulation_materials = [ - # {'type': 'ewi_wall_demolition', - # 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' - # 'hammer; plaster to walls.', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, - # 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, - # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', - # 'description': 'Stud walls: Remove wall linings ' - # 'including battening behind; ' - # 'plasterboard and skim', - # 'depth': 0, 'depth_unit': 0, - # 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, - # 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, - # 'plant_cost': 1.25, 'total_cost': 7.48, - # 'link': 'SPONs', 'Notes': 0}, - # {'type': 'ewi_wall_demolition', - # 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' - # 'behind; wood lath and plaster', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - # 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, - # 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, - # 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', - # 'description': 'Clean and prepare surfaces, ' - # 'one coat Keim dilution, ' - # 'one coat primer and two coats ' - # 'of Keim Ecosil paint; Brick or ' - # 'block walls; over 300 mm girth', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, - # 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, - # 'prime_material_cost': 0, 'material_cost': 7.3, - # 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, - # 'plant_cost': 0, 'total_cost': 12.92, - # 'link': 'SPONs', - # 'Notes': 'This work covers the preparation and ' - # 'priming of the wall before insulating'}, - # {'type': 'ewi_wall_redecoration', - # 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' - # 'separately) with horizontal PVC intermediate track and vertical ' - # 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' - # 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' - # 'Topcoat Render (white)', - # 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - # 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, - # 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, - # 'total_cost': 69.94, 'link': 'SPONs', - # 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' - # 'cost of 99.17 per meter square. This includes the cost of insulation. ' - # 'To get the costing for just the works and not the insulation, ' - # 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' - # 'as an example, which costs £29.23 per meter square, giving us the cost ' - # 'of the remaining works without insulation. This material gives us a ' - # 'cost for basecoat, mesh application and a render finish'}] - demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] @@ -759,5 +593,6 @@ class Costs: "material": materials_costs, "profit": profit_cost, "labour_hours": labour_hours, - "labour_days": labour_days + "labour_days": labour_days, + "labour_cost": labour_costs } diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 8e2d76ce..1ba601a8 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -30,7 +30,7 @@ class TestCosts: assert cwi_results == {'total': 1027.0280465530302, 'subtotal': 855.8567054608585, 'vat': 171.1713410921717, 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574, 'profit': 95.09518949565093, - 'labour_hours': 6.234177828761786} + 'labour_hours': 6.234177828761786, 'labour_cost': 94.95132385344874} def test_loft_insulation(self): mock_property = Mock() @@ -56,7 +56,8 @@ class TestCosts: assert loft_results == {'total': 414.8496486, 'subtotal': 345.70804050000004, 'vat': 69.14160810000001, 'contingency': 25.608003000000004, 'preliminaries': 25.608003000000004, - 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685} + 'material': 198.29923000000002, 'profit': 38.4120045, 'labour_hours': 3.685, + 'labour_cost': 57.7808} def test_internal_wall_insulation(self): mock_property = Mock() @@ -173,4 +174,238 @@ class TestCosts: assert iwi_results == {'total': 6421.5484411659245, 'subtotal': 5351.29036763827, 'vat': 1070.258073527654, 'contingency': 573.3525393898148, 'preliminaries': 382.2350262598765, 'material': 1747.488000615996, 'profit': 573.3525393898148, - 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405} + 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405, + 'labour_cost': 1927.1602026551818} + + def test_suspended_floor_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + + sus_floor_material = {'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', + 'depth': 140.0, + 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.039, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, + 'plant_cost': 0, + 'total_cost': 13.46, 'link': 'SPONs', + 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. ' + 'We use the ' + 'same values as in Crown loft roll 44, since it is also an insulation roll'} + + sus_floor_non_insulation_materials = [ + {'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'suspended_floor_demolition', + 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, + 'plant_cost': 0, 'total_cost': 9.34, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 1.54, 'labour_cost': 24.98, + 'labour_hours_per_unit': 0.74, 'plant_cost': 0, 'total_cost': 26.52, 'link': 'SPONs', 'Notes': 0}, + {'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}] + + sus_floor_results = costs.suspended_floor_insulation( + insulation_floor_area=33.5, + material=sus_floor_material, + non_insulation_materials=sus_floor_non_insulation_materials + ) + + assert sus_floor_results == { + 'total': 3003.366924, 'subtotal': 2502.80577, 'vat': 500.561154, + 'contingency': 185.39302, 'preliminaries': 185.39302, 'material': 483.405, + 'profit': 278.08952999999997, 'labour_hours': 54.940000000000005, + 'labour_days': 2.289166666666667, 'labour_cost': 1370.5252 + } + + def test_solid_floor_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire" + } + + costs = Costs(mock_property) + sol_floor_material = { + 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, + 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, + 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + } + + sol_floor_non_insulation_materials = [ + {'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, + 'plant_cost': 0, 'total_cost': 3.32, 'link': 'SPONs', + 'Notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and ' + 'therefore there is no need for a skip'}, + {'type': 'solid_floor_preparation', + 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, + 'plant_cost': 0, 'total_cost': 4.36, 'link': 0, 'Notes': 0}, {'type': 'solid_floor_preparation', + 'description': 'Clean out crack to ' + 'form a 20mm×20mm ' + 'groove and fill with ' + 'cement: mortar mixed ' + 'with bonding agent', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, + 'material_cost': 6.91, + 'labour_cost': 18.99, + 'labour_hours_per_unit': 0.61, + 'plant_cost': 0.16, + 'total_cost': 26.06, 'link': 0, + 'Notes': 'This step is the ' + 'assessment and repair of ' + 'any damage to the concrete ' + 'floor such as filling ' + 'cracks or levelling uneven ' + 'areas'}, + {'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, + 'labour_hours_per_unit': 0.02, 'plant_cost': 0, 'total_cost': 1.69, 'link': 'SPONs', 'Notes': 0}, + {'type': 'solid_floor_redecoration', + 'description': 'Screeded beds; protection to compressible formwork exceeding 600mm wide', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 9.6, 'material_cost': 9.89, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, + 'plant_cost': 0, 'total_cost': 12.56, 'link': 'SPONs', + 'Notes': 'This is the screed layer, placed on top of the insulation'}, + {'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0, 'depth_unit': 0, + 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, + 'plant_cost': 0, 'total_cost': 6.59, 'link': 'SPONs', + 'Notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; ' + 'Gradus woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, ' + 'therefore we need just labour rates'}, + {'type': 'solid_floor_redecoration', + 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0, + 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, + 'plant_cost': 0, 'total_cost': 4.88, 'link': 'SPONs', 'Notes': 0} + ] + + sol_floor_results = costs.solid_floor_insulation( + insulation_floor_area=33.5, + material=sol_floor_material, + non_insulation_materials=sol_floor_non_insulation_materials + ) + + assert sol_floor_results == { + 'total': 3962.021952, 'subtotal': 3301.68496, 'vat': 660.336992, 'contingency': 353.75196, + 'preliminaries': 235.83464, 'material': 1006.3399999999999, 'profit': 353.75196, 'labour_hours': 57.285, + 'labour_days': 2.386875, 'labour_cost': 1346.6464 + } + + def test_external_wall_insulation(self): + mock_property = Mock() + mock_property.data = { + "county": "Northamptonshire", + "property-type": "House", + "built-form": 'End-Terrace' + } + + costs = Costs(mock_property) + + ewi_material = {'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', + 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, + 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, + 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0} + ewi_non_insulation_materials = [ + {'type': 'ewi_wall_demolition', + 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' + 'hammer; plaster to walls.', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, + 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, + 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', + 'description': 'Stud walls: Remove wall linings ' + 'including battening behind; ' + 'plasterboard and skim', + 'depth': 0, 'depth_unit': 0, + 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, + 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, + 'plant_cost': 1.25, 'total_cost': 7.48, + 'link': 'SPONs', 'Notes': 0}, + {'type': 'ewi_wall_demolition', + 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' + 'behind; wood lath and plaster', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', + 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, + 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, + 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', + 'description': 'Clean and prepare surfaces, ' + 'one coat Keim dilution, ' + 'one coat primer and two coats ' + 'of Keim Ecosil paint; Brick or ' + 'block walls; over 300 mm girth', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, + 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, + 'prime_material_cost': 0, 'material_cost': 7.3, + 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, + 'plant_cost': 0, 'total_cost': 12.92, + 'link': 'SPONs', + 'Notes': 'This work covers the preparation and ' + 'priming of the wall before insulating'}, + {'type': 'ewi_wall_redecoration', + 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' + 'separately) with horizontal PVC intermediate track and vertical ' + 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' + 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' + 'Topcoat Render (white)', + 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, + 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, + 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, + 'total_cost': 69.94, 'link': 'SPONs', + 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' + 'cost of 99.17 per meter square. This includes the cost of insulation. ' + 'To get the costing for just the works and not the insulation, ' + 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' + 'as an example, which costs £29.23 per meter square, giving us the cost ' + 'of the remaining works without insulation. This material gives us a ' + 'cost for basecoat, mesh application and a render finish'}] + + ewi_results = costs.external_wall_insulation( + wall_area=95.9104281347967, + material=ewi_material, + non_insulation_materials=ewi_non_insulation_materials + ) + + assert ewi_results == { + 'total': 13590.909723215433, 'subtotal': 11325.758102679527, 'vat': 2265.1516205359053, + 'contingency': 808.9827216199662, 'preliminaries': 1213.4740824299492, + 'material': 4020.565147410677, 'profit': 1213.4740824299492, + 'labour_hours': 187.02533486285358, 'labour_days': 5.8445417144641745, + 'labour_cost': 3921.5600094613983 + } From b8ae3450761f8efd4cba7c38aace55974ed6a7d8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 19:17:58 +0000 Subject: [PATCH 20/27] Created costs elt app --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 29 ++++++++- etl/costs/README.md | 35 +++++++++++ etl/costs/app.py | 98 ++++++++++++++++++++++++++++++ etl/costs/requirements.txt | 5 ++ 6 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 etl/costs/README.md create mode 100644 etl/costs/app.py create mode 100644 etl/costs/requirements.txt diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..ed9033de 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..3ab974fc 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 812c1ebb..e191c5ee 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -19,6 +19,20 @@ class MaterialType(enum.Enum): flat_roof_insulation = "flat_roof_insulation" room_roof_insulation = "room_roof_insulation" + iwi_wall_demolition = "iwi_wall_demolition" + iwi_vapour_barrier = "iwi_vapour_barrier" + iwi_redecoration = "iwi_redecoration" + suspended_floor_demolition = "suspended_floor_demolition" + suspended_floor_redecoration = "suspended_floor_redecoration" + suspended_floor_vapour_barrier = "suspended_floor_vapour_barrier" + solid_floor_demolition = "solid_floor_demolition" + solid_floor_preparation = "solid_floor_preparation" + solid_floor_vapour_barrier = "solid_floor_vapour_barrier" + solid_floor_redecoration = "solid_floor_redecoration" + ewi_wall_demolition = "ewi_wall_demolition" + ewi_wall_preparation = "ewi_wall_preparation" + ewi_wall_redecoration = "ewi_wall_redecoration" + class DepthUnit(enum.Enum): mm = "mm" @@ -27,6 +41,7 @@ class DepthUnit(enum.Enum): class CostUnit(enum.Enum): gbp_sq_meter = "gbp_sq_meter" gbp_per_unit = "gbp_per_unit" + gbp_per_m2 = "gbp_per_m2" class RValueUnit(enum.Enum): @@ -41,9 +56,11 @@ class Material(Base): __tablename__ = 'material' id = Column(Integer, primary_key=True, autoincrement=True) - type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x]), nullable=False) + type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x], create_constraint=False), + nullable=False) + description = Column(String, nullable=False) - depths = Column(String) # You may want to use a specific JSON type depending on the database + depth = Column(String) # You may want to use a specific JSON type depending on the database depth_unit = Column(Enum(DepthUnit, values_callable=lambda x: [e.value for e in x]), nullable=False) cost = Column(String) cost_unit = Column(Enum(CostUnit, values_callable=lambda x: [e.value for e in x]), nullable=False) @@ -57,3 +74,11 @@ class Material(Base): link = Column(String) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) is_active = Column(Boolean, nullable=False, default=True) + + prime_material_cost = Column(Float) + material_cost = Column(Float) + labour_cost = Column(Float) + labour_hours_per_unit = Column(Float) + plant_cost = Column(Float) + total_cost = Column(Float) + notes = Column(String) diff --git a/etl/costs/README.md b/etl/costs/README.md new file mode 100644 index 00000000..969a3173 --- /dev/null +++ b/etl/costs/README.md @@ -0,0 +1,35 @@ +### Costs ETL Application + +This is a simple application to push the materials costs data to the database. + +#### How to run + +Ensure you have a .env file in the base Model directory with the following variables + +``` +DB_HOST="Your db host" +DB_PORT="Your db port" +DB_USER="Your db user" +DB_PASSWORD="Your db password" +DB_NAME="Your db name" +``` + +Make sure your python path environment variable pouints to the base Model directory. To set the +`PYTHONPATH` environment variable, run the following command from the base Model directory + +``` +export PYTHONPATH=`pwd` +``` + +From the base Model directory, install the requirements by running the following command + +``` +pip install -r etl/costs/requirements.txt +``` + +Then run the following command to run the application + +``` +python etl/costs/app.py +``` + diff --git a/etl/costs/app.py b/etl/costs/app.py new file mode 100644 index 00000000..0117a66e --- /dev/null +++ b/etl/costs/app.py @@ -0,0 +1,98 @@ +import os +import dotenv +import json +import pandas as pd +import numpy as np +from pathlib import Path +from sqlalchemy.orm import Session +from sqlalchemy import create_engine +from backend.app.db.models.materials import Material + +DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx" +# Environment file is at the same level as this file +ENV_FILE = Path(__file__).parent / "etl" / "costs" / ".env" +dotenv.load_dotenv(ENV_FILE) + +DB_USERNAME = os.getenv('DB_USERNAME') +DB_PASSWORD = os.getenv('DB_PASSWORD') +DB_HOST = os.getenv('DB_HOST') +DB_PORT = os.getenv('DB_PORT') +DB_NAME = os.getenv('DB_NAME') + + +def push_costs_to_db(engine, costs_df): + """ + Push costs DataFrame to the database. + + :param engine: The SQLAlchemy engine connected to your database. + :param costs_df: The DataFrame containing cost data. + """ + materials = [] + + for _, row in costs_df.iterrows(): + row_dict = row.to_dict() + + # Add other necessary transformations here + + # Create Material object and add it to the list + materials.append(Material(**row_dict)) + + # Use SQLAlchemy session for bulk insert + with Session(engine) as session: + session.bulk_save_objects(materials) + session.commit() + + +def app(): + """ + This application uploads the cost data to our database + + The most recent cost data can be found in OneDrive, in the + shared folder > 04. Product Development > Cost data > Hestia Materials.xlsx + + For the moment, the data is uploaded manually. In the future, we will automate this so the data can be + stored locally and then is uploaded from the local_data folder + :return: + """ + + connection_string = "postgresql+{drivername}://{username}:{password}@{server}:{port}/{dbname}" + db_string = connection_string.format( + drivername="psycopg2", # You'll need to use psycopg2 driver for PostgreSQL + username=DB_USERNAME, + password=DB_PASSWORD, + server=DB_HOST, + port=DB_PORT, + dbname=DB_NAME, + ) + + db_engine = create_engine(db_string, pool_size=5, max_overflow=5) + + cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0) + loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0) + iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0) + suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0) + solid_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="solid_floor_insulation", header=0) + ewi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="external_wall_insulation", header=0) + + # Form a single table to be uploaded + costs = pd.concat( + [ + cwi_costs, + loft_insulation_costs, + iwi_costs, + suspended_floor_costs, + solid_floor_costs, + ewi_costs, + ] + ) + + costs = costs.replace({np.nan: None}) + costs["depth"] = costs["depth"].fillna(0) + costs["depth"] = costs["depth"].astype(str) + + # Push the costs to the database + push_costs_to_db(db_engine, costs) + + +if __name__ == "__main__": + app() diff --git a/etl/costs/requirements.txt b/etl/costs/requirements.txt new file mode 100644 index 00000000..7d6afa9e --- /dev/null +++ b/etl/costs/requirements.txt @@ -0,0 +1,5 @@ +pandas==1.5.3 +sqlalchemy==2.0.19 +python-dotenv +psycopg2-binary +openpyxl \ No newline at end of file From 79ddd648274688d9ad9845bc2e4b5555937c6f91 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 23 Nov 2023 19:44:44 +0000 Subject: [PATCH 21/27] deleted temp data from costs --- recommendations/Costs.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 8f54a79a..c1c9b42e 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1,32 +1,5 @@ import numpy as np -# Example - delete me -from backend.Property import Property - -print("DELETE ME IN COSTS CLASS") -from epc_api.client import EpcClient - -epc_client = EpcClient(auth_token=AUTH_TOKEN) -p1 = Property( - postcode="NN1 5JY", - address1="2 South Terrace", - epc_client=epc_client, - id=0 -) - -p2 = Property( - postcode="PO12 4TY", - address1="25 Albert Street", - epc_client=epc_client, - id=0 -) - -p1.search_address_epc() -p2.search_address_epc() - -p1.set_basic_property_dimensions() -p2.set_basic_property_dimensions() - # This data comes from SPONs regional_labour_variations = [ {"Region": "Outer London (Spon’s 2023)", "Adjustment_Factor": 1.00}, From 96553d0fc01b4ced14fc4ff45015da3e69a77de6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:00:35 +0000 Subject: [PATCH 22/27] Updating the floor recommendations class for new cost data --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 7 +- backend/app/plan/utils.py | 26 +++----- etl/costs/app.py | 10 ++- recommendations/FloorRecommendations.py | 88 ++++++++++++++++++------- recommendations/recommendation_utils.py | 12 ++-- 7 files changed, 93 insertions(+), 54 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index ed9033de..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 3ab974fc..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 83a57d07..f110b27a 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import ( - create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id + create_recommendation_scoring_data, prepare_materials, get_cleaned, insert_temp_recommendation_id ) from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 @@ -114,7 +114,7 @@ async def trigger_plan(body: PlanTriggerRequest): # the same data logger.info("Reading in materials and cleaned datasets") materials = get_materials(session) - materials_by_type = filter_materials(materials) + materials = prepare_materials(materials) cleaned = get_cleaned() logger.info("Getting components and epc recommendations") @@ -126,13 +126,14 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations_scoring_data = [] for p in input_properties: + # Property recommendations p.get_components(cleaned) property_recommendations = [] # Floor recommendations - floor_recommender = FloorRecommendations(property_instance=p, materials=materials_by_type["floor"]) + floor_recommender = FloorRecommendations(property_instance=p, materials=materials) floor_recommender.recommend() if floor_recommender.recommendations: diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index e2bf9d86..e3723b24 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -1,6 +1,5 @@ import pandas as pd from backend.Property import Property -from collections import defaultdict from utils.s3 import read_from_s3 from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value @@ -10,22 +9,13 @@ from backend.app.config import get_settings import msgpack -def filter_materials(materials): - materials_by_type = defaultdict(list) - - mapping = { - "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"], - "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"], - "ventilation": ["mechanical_ventilation"], - "roof": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"] - } - - materials = [row2dict(material) for material in materials] - - for component, types in mapping.items(): - materials_by_type[component] = [part for part in materials if part["type"] in types] - - return dict(materials_by_type) +def prepare_materials(materials): + """ + This function will prepare the materials for recommendations + :param materials: list of materials, as retrieved from the database + :return: + """ + return [row2dict(material) for material in materials] def insert_temp_recommendation_id(property_recommendations): @@ -173,7 +163,7 @@ def create_recommendation_scoring_data( parts = recommendation["parts"] if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - + scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: diff --git a/etl/costs/app.py b/etl/costs/app.py index 0117a66e..1ecbbb5f 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -1,12 +1,12 @@ import os import dotenv -import json import pandas as pd import numpy as np from pathlib import Path from sqlalchemy.orm import Session from sqlalchemy import create_engine from backend.app.db.models.materials import Material +from recommendations.recommendation_utils import calculate_r_value_per_mm DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx" # Environment file is at the same level as this file @@ -90,6 +90,14 @@ def app(): costs["depth"] = costs["depth"].fillna(0) costs["depth"] = costs["depth"].astype(str) + costs["r_value_per_mm"] = costs.apply( + lambda row: calculate_r_value_per_mm(float(row["depth"]), row["thermal_conductivity"]), axis=1 + ) + costs["r_value_unit"] = "square_meter_kelvin_per_watt" + + for col in ["material_cost", "labour_cost", "labour_hours_per_unit", "plant_cost"]: + costs[col] = costs[col].fillna(0) + # Push the costs to the database push_costs_to_db(db_engine, costs) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 5b194e0d..641272a3 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -1,5 +1,8 @@ import math from typing import List + +import pandas as pd + from BaseUtility import Definitions from datatypes.enums import QuantityUnits from backend.Property import Property @@ -8,6 +11,7 @@ from recommendations.recommendation_utils import ( get_recommended_part, get_floor_u_value ) from recommendations.rdsap_tables import FLOOR_LEVEL_MAP +from recommendations.Costs import Costs class FloorRecommendations(Definitions): @@ -30,25 +34,41 @@ class FloorRecommendations(Definitions): materials: List, ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials - - self.suspended_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "suspended_floor_insulation" - ] - self.solid_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "solid_floor_insulation" + self.suspended_floor_insulation_materials = [ + part for part in materials if part["type"] == "suspended_floor_insulation" ] - self.exposed_floor_insulation_parts = [ - part for part in self.materials if part["type"] == "exposed_floor_insulation" + self.suspended_floor_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "suspended_floor_demolition", "suspended_floor_redecoration", "suspended_floor_vapour_barrier" + ] ] + self.solid_floor_insulation_materials = [ + part for part in materials if part["type"] == "solid_floor_insulation" + ] + + self.solid_floor_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "solid_floor_demolition", "solid_floor_preparation", "solid_floor_vapour_barrier", + "solid_floor_redecoration" + ] + ] + + self.exposed_floor_insulation_materials = [ + part for part in materials if part["type"] == "exposed_floor_insulation" + ] + + # TODO: To be completed + self.exposed_floor_non_insulation_materials = [] + def recommend(self): u_value = self.property.floor["thermal_transmittance"] @@ -98,7 +118,11 @@ class FloorRecommendations(Definitions): if self.property.floor["is_suspended"]: # Given the U-value, we recommend underfloor insulation - self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts) + self.recommend_floor_insulation( + u_value=u_value, + insulation_materials=self.suspended_floor_insulation_materials, + non_insulation_materials=self.suspended_floor_non_insulation_materials + ) return if self.property.floor["is_solid"]: @@ -113,20 +137,23 @@ class FloorRecommendations(Definitions): raise NotImplementedError("Implement me!") @staticmethod - def _make_floor_description(part, depth): - return f"Install {depth}{part['depth_unit']} {part['description']} insulation" + def _make_floor_description(material): + return f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation" - def recommend_floor_insulation(self, u_value, parts): + def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials): """ This method is tasked with estimating the impact of performing suspended floor insulation :return: """ - lowest_selected_u_value = None - for part in parts: - for depth, cost_per_unit in zip(part["depths"], part["cost"]): + insulation_materials = pd.DataFrame(insulation_materials) - part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"]) + lowest_selected_u_value = None + for _, insulation_material_group in insulation_materials.groupby("description"): + + for _, material in insulation_material_group.iterrows(): + + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -137,26 +164,37 @@ class FloorRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - quantity = self.property.insulation_floor_area - estimated_cost = cost_per_unit * quantity + if material["type"] == "suspended_floor_insulation": + cost_result = self.costs.suspended_floor_insulation( + insulation_floor_area=self.property.insulation_floor_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + elif material["type"] == "solid_floor_insulation": + cost_result = self.costs.solid_floor_insulation( + insulation_floor_area=self.property.insulation_floor_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + else: + raise NotImplementedError("Implement me!") self.recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=depth, - quantity=quantity, + part=material.to_dict(), + quantity=self.property.insulation_floor_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ), ], "type": "floor_insulation", - "description": self._make_floor_description(part, depth), + "description": self._make_floor_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 217f313f..5bd77a2a 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -109,22 +109,21 @@ def update_lowest_selected_u_value(lowest_selected_u_value, new_u_value): return lowest_selected_u_value -def get_recommended_part(part, selected_depth, selected_total_cost, quantity, quantity_unit): +def get_recommended_part(part, cost_result, quantity, quantity_unit): """ Utility function to return a recommended part with the selected depth. :param part: part to be recommended - :param selected_depth: depth of the selected part - :param selected_total_cost: Total cost of the selected part + :param cost_result: Total cost of the selected part, as returned by the Cost class :param quantity: Quantity of the selected part :param quantity_unit: Unit of the quantity :return: """ recommended_part = deepcopy(part) - recommended_part["depths"] = [selected_depth] - recommended_part["estimated_cost"] = selected_total_cost recommended_part["quantity"] = quantity recommended_part["quantity_unit"] = quantity_unit + recommended_part.update(cost_result) + return recommended_part @@ -563,6 +562,9 @@ def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK): :return: """ + if thermal_conductivity_w_mK is None: + return None + r_value_m2k_w = (thickness_mm / 1000) / thermal_conductivity_w_mK # Calculate R-value per mm From 80b4d2390c092af1a59ece21995d2cf67d9bb44a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:03:09 +0000 Subject: [PATCH 23/27] update floor recommendations for suspended floor --- backend/app/db/functions/materials_functions.py | 5 ++++- backend/app/plan/router.py | 5 ++--- backend/app/plan/utils.py | 10 ---------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/backend/app/db/functions/materials_functions.py b/backend/app/db/functions/materials_functions.py index f3c2f316..4d492946 100644 --- a/backend/app/db/functions/materials_functions.py +++ b/backend/app/db/functions/materials_functions.py @@ -1,4 +1,5 @@ from backend.app.db.models.materials import Material +from backend.app.db.utils import row2dict from functools import lru_cache @@ -16,4 +17,6 @@ def get_materials(session): materials = session.query(Material).filter(Material.is_active).all() - return materials if materials else [] + materials = materials if materials else [] + + return [row2dict(material) for material in materials] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index f110b27a..8d8ffe2d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import ( - create_recommendation_scoring_data, prepare_materials, get_cleaned, insert_temp_recommendation_id + create_recommendation_scoring_data, get_cleaned, insert_temp_recommendation_id ) from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 @@ -114,7 +114,6 @@ async def trigger_plan(body: PlanTriggerRequest): # the same data logger.info("Reading in materials and cleaned datasets") materials = get_materials(session) - materials = prepare_materials(materials) cleaned = get_cleaned() logger.info("Getting components and epc recommendations") @@ -141,7 +140,7 @@ async def trigger_plan(body: PlanTriggerRequest): # Wall recommendations - wall_recomender = WallRecommendations(property_instance=p, materials=materials_by_type["walls"]) + wall_recomender = WallRecommendations(property_instance=p, materials=materials) wall_recomender.recommend() if wall_recomender.recommendations: diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index e3723b24..b73ba874 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -4,20 +4,10 @@ from utils.s3 import read_from_s3 from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value -from backend.app.db.utils import row2dict from backend.app.config import get_settings import msgpack -def prepare_materials(materials): - """ - This function will prepare the materials for recommendations - :param materials: list of materials, as retrieved from the database - :return: - """ - return [row2dict(material) for material in materials] - - def insert_temp_recommendation_id(property_recommendations): """ Creates a temporary recommendation id which is needed for From cb52c9f7a3b2fe443f1bf08532ba3291bc639bcc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 08:15:32 +0000 Subject: [PATCH 24/27] Updated wall recommended partially for new costs --- recommendations/WallRecommendations.py | 88 +++++++++++++++++++------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 12085840..4595ef22 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -1,6 +1,8 @@ import math from typing import List +import pandas as pd + from datatypes.enums import QuantityUnits from backend.Property import Property from BaseUtility import Definitions @@ -9,6 +11,7 @@ from recommendations.recommendation_utils import ( get_recommended_part, get_wall_u_value ) from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION +from recommendations.Costs import Costs from utils.logger import setup_logger logger = setup_logger() @@ -50,13 +53,36 @@ class WallRecommendations(Definitions): materials: List ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials + self.cavity_wall_insulation_materials = [ + part for part in materials if part["type"] == "cavity_wall_insulation" + ] + + self.internal_wall_insulation_materials = [ + part for part in materials if part["type"] == "internal_wall_insulation" + ] + + self.internal_wall_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration" + ] + ] + + self.external_wall_insulation_materials = [ + part for part in materials if part["type"] == "external_wall_insulation" + ] + + self.external_wall_non_insulation_materials = [ + part for part in materials if part["type"] in [ + "ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration" + ] + ] @property def ewi_valid(self): @@ -200,15 +226,15 @@ class WallRecommendations(Definitions): self.recommendations = recommendations - def _find_insulation(self, parts, u_value): + def _find_insulation(self, u_value, insulation_materials, non_insulation_materials): + lowest_selected_u_value = None recommendations = [] - for part in parts: + for _, insulation_material_group in insulation_materials.groupby("description"): - for depth, cost_per_unit in zip(part["depths"], part["cost"]): - - part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"]) + for _, material in insulation_material_group.iterrows(): + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -225,27 +251,40 @@ class WallRecommendations(Definitions): # We allow a small tolerance for error so we don't discount the recommendation entirely if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = cost_per_unit * self.property.insulation_wall_area + if material["type"] == "internal_wall_insulation": + cost_result = self.costs.internal_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + elif material["type"] == "external_wall_insulation": + cost_result = self.costs.external_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + non_insulation_materials=non_insulation_materials + ) + else: + raise ValueError("Invalid material type") recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=depth, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "wall_insulation", - "description": "Install " + self._make_description(part, depth), + "description": "Install " + self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) @@ -258,27 +297,32 @@ class WallRecommendations(Definitions): :return: """ - ewi_parts = [ - part for part in self.materials if part["type"] == "external_wall_insulation" - ] if self.ewi_valid else [] - - iwi_parts = [part for part in self.materials if part["type"] == "internal_wall_insulation"] - # Recommend external and internal wall insulation separately # Since external and internal wall insulation are sufficiently different, # we separate the logic for for recommending them, therefore we don't # consider diminishing returns between the two - ewi_recommendations = self._find_insulation(ewi_parts, u_value) - iwi_recommendations = self._find_insulation(iwi_parts, u_value) + ewi_recommendations = [] + if self.ewi_valid: + ewi_recommendations = self._find_insulation( + u_value=u_value, + insulation_materials=pd.DataFrame(self.external_wall_insulation_materials), + non_insulation_materials=self.external_wall_non_insulation_materials + ) + + iwi_recommendations = self._find_insulation( + u_value=u_value, + insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials), + non_insulation_materials=self.internal_wall_non_insulation_materials + ) self.recommendations += ewi_recommendations + iwi_recommendations self.prune_diminishing_recommendations() @staticmethod - def _make_description(part, depth): - return f"{depth}{part['depth_unit']} {part['description']}" + def _make_description(material): + return f"{int(material['depth'])}{material['depth_unit']} {material['description']}" def prune_diminishing_recommendations(self): # For any recommendations, if we have at least 1 reommendation that does not exhibit diminishing returns From bf2e6c1ebc68771b17c4ffa2dbd2a43490fb36c2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 09:58:13 +0000 Subject: [PATCH 25/27] Updating recommendation classes with new cost data --- backend/app/plan/router.py | 4 +- backend/app/plan/utils.py | 2 +- recommendations/Costs.py | 4 +- recommendations/FireplaceRecommendations.py | 2 +- recommendations/FloorRecommendations.py | 6 +- recommendations/RoofRecommendations.py | 67 ++++++++++++------- recommendations/VentilationRecommendations.py | 2 +- recommendations/WallRecommendations.py | 21 +++--- 8 files changed, 67 insertions(+), 41 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8d8ffe2d..fffc604e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -147,7 +147,7 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations.append(wall_recomender.recommendations) # Roof recommendations - roof_recommender = RoofRecommendations(property_instance=p, materials=materials_by_type["roof"]) + roof_recommender = RoofRecommendations(property_instance=p, materials=materials) roof_recommender.recommend() if roof_recommender.recommendations: @@ -156,7 +156,7 @@ async def trigger_plan(body: PlanTriggerRequest): # Ventilation recommendations ventilation_recomender = VentilationRecommendations( property_instance=p, - materials=materials_by_type["ventilation"] + materials=[part for part in materials if part["type"] == "mechanical_ventilation"] ) ventilation_recomender.recommend() diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index b73ba874..20b5db5b 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -154,7 +154,7 @@ def create_recommendation_scoring_data( if len(parts) != 1: raise ValueError("More than one part for roof insulation - investiage me") - scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0]) + scoring_dict["roof_insulation_thickness_ENDING"] = str(int(parts[0]["depth"])) scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good" else: # Fill missing roof u-values - this fill is not based on recommended upgrades diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c1c9b42e..a96e1215 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -113,7 +113,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * wall_area + labour_hours = material["labour_hours_per_unit"] * wall_area return { "total": total_cost, @@ -151,7 +151,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost - labour_hours = material["labour_hours"] * floor_area + labour_hours = material["labour_hours_per_unit"] * floor_area return { "total": total_cost, diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 3e82b9d1..9524c75a 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -43,6 +43,6 @@ class FireplaceRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, - "cost": estimated_cost, + "total": estimated_cost, } ] diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 641272a3..96b1356c 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -127,7 +127,11 @@ class FloorRecommendations(Definitions): if self.property.floor["is_solid"]: # Given the U-value, we recommend solid floor insulation options which are usually solid foam - self.recommend_floor_insulation(u_value=u_value, parts=self.solid_floor_insulation_parts) + self.recommend_floor_insulation( + u_value=u_value, + insulation_materials=self.solid_floor_insulation_materials, + non_insulation_materials=self.solid_floor_non_insulation_materials + ) return if self.property.floor["is_to_unheated_space"] or self.property.floor["is_to_external_air"]: diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 4f96f629..1bee1e8e 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -1,4 +1,5 @@ import math +import pandas as pd from backend.Property import Property from typing import List from datatypes.enums import QuantityUnits @@ -6,6 +7,7 @@ from recommendations.recommendation_utils import ( get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric ) +from recommendations.Costs import Costs class RoofRecommendations: @@ -27,13 +29,17 @@ class RoofRecommendations: materials: List ): self.property = property_instance + self.costs = Costs(self.property) # For audit purposes, when estimating u values we'll store it self.estimated_u_value = None # Will contains a list of recommended measures self.recommendations = [] - self.materials = materials + self.loft_insulation_materials = [ + part for part in materials if part["type"] == "loft_insulation" + ] + self.loft_non_insulation_materials = [] def recommend(self): @@ -58,7 +64,7 @@ class RoofRecommendations: # If we have a u-value already, need to implement this if u_value: if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - # The floor is already compliant + # The Roof is already compliant return if self.property.data["transaction-type"] == "new dwelling": @@ -66,6 +72,10 @@ class RoofRecommendations: raise NotImplementedError("Implement me") u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band}) + self.estimated_u_value = u_value + if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + # The Roof is already compliant + return if self.property.roof["is_pitched"] or self.property.roof["is_flat"]: self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof) @@ -78,18 +88,21 @@ class RoofRecommendations: raise NotImplementedError("Implement me") @staticmethod - def make_loft_insulation_description(material, depth): - return f"Install {depth}{material['depth_unit']} of {material['description']} in your loft" + def make_loft_insulation_description(material): + return f"Install {int(material['depth'])}{material['depth_unit']} of {material['description']} in your loft" @staticmethod def make_room_roof_insulation_description(material, depth): return f"Insulate your room roof with {depth}{material['depth_unit']} of {material['description']}" @staticmethod - def make_flat_roof_insulation_description(material, depth): - return f"Insulate the home's flat roof with {depth}{material['depth_unit']} of {material['description']}" + def make_flat_roof_insulation_description(material): + return (f"Insulate the home's flat roof " + f"with {int(material['depth'])}{material['depth_unit']} of {material['description']}") - def recommend_roof_insulation(self, u_value, insulation_thickness, roof): + def recommend_roof_insulation( + self, u_value, insulation_thickness, roof + ): """ This method will recommend which insulation materials to use @@ -120,28 +133,31 @@ class RoofRecommendations: # from the base layer if roof["is_pitched"]: - materials = [m for m in self.materials if m["type"] == "loft_insulation"] + insulation_materials = self.loft_insulation_materials + non_insulation_materials = self.loft_non_insulation_materials elif roof["is_flat"]: - materials = [m for m in self.materials if m["type"] == "flat_roof_insulation"] + raise ValueError("UPDATE ME") else: raise ValueError("Roof is not pitched or flat") - if not materials: + if not insulation_materials: raise ValueError("No roof insulation materials found") + insulation_materials = pd.DataFrame(insulation_materials) + lowest_selected_u_value = None recommendations = [] - for material in materials: + for _, insulation_material_group in insulation_materials.groupby("description"): - for depth, cost_per_unit in zip(material["depths"], material["cost"]): + for _, material in insulation_material_group.iterrows(): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the # loft is already partially insulated. # Note: This requirement is only for loft insulation - if ((depth + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: + if ((material["depth"] + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]: continue - part_u_value = r_value_per_mm_to_u_value(depth, material["r_value_per_mm"]) + part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -161,23 +177,26 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - # TODO: We should use the floor area divided by the number of floors to get the area of the roof - estimated_cost = cost_per_unit * self.property.floor_area - - if roof["is_pitched"]: - description = self.make_loft_insulation_description(material, depth) + if material["type"] == "loft_insulation": + cost_result = self.costs.loft_insulation( + floor_area=self.property.insulation_floor_area, + material=material + ) + description = self.make_loft_insulation_description(material) + elif material["type"] == "flat_roof_insulation": + description = self.make_flat_roof_insulation_description(material) + raise ValueError("COMPLETE ME") else: - description = self.make_flat_roof_insulation_description(material, depth) + raise ValueError("Invalid material type") recommendations.append( { "parts": [ get_recommended_part( - part=material, - selected_depth=depth, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "roof_insulation", @@ -185,7 +204,7 @@ class RoofRecommendations: "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index a639905b..a0b188f7 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -65,6 +65,6 @@ class VentilationRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": None, - "cost": estimated_cost, + "total": estimated_cost, } ] diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 4595ef22..acc74ead 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -180,7 +180,7 @@ class WallRecommendations(Definitions): filled cavity wall """ - cavity_wall_fills = [m for m in self.materials if m["type"] == "cavity_wall_insulation"] + insulation_materials = pd.DataFrame(self.cavity_wall_insulation_materials) cavity_width = 75 if insulation_thickness == "below average": cavity_width = cavity_width * (1 - PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION) @@ -188,8 +188,9 @@ class WallRecommendations(Definitions): # Test the different fill options lowest_selected_u_value = None recommendations = [] - for part in cavity_wall_fills: - part_u_value = r_value_per_mm_to_u_value(cavity_width, part["r_value_per_mm"]) + for _, material in insulation_materials.iterrows(): + + part_u_value = r_value_per_mm_to_u_value(cavity_width, material["r_value_per_mm"]) _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 @@ -202,25 +203,27 @@ class WallRecommendations(Definitions): if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) - estimated_cost = part["cost"] * self.property.insulation_wall_area + cost_result = self.costs.cavity_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + ) recommendations.append( { "parts": [ get_recommended_part( - part=part, - selected_depth=None, + part=material.to_dict(), quantity=self.property.insulation_wall_area, quantity_unit=QuantityUnits.m2.value, - selected_total_cost=estimated_cost + cost_result=cost_result ) ], "type": "wall_insulation", - "description": f"Fill cavity with {part['description']}", + "description": f"Fill cavity with {material['description']}", "starting_u_value": u_value, "new_u_value": new_u_value, "sap_points": None, - "cost": estimated_cost, + **cost_result } ) From 02b76cd9fe6e24a30808352e6642799f11bbaacd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 11:01:44 +0000 Subject: [PATCH 26/27] Updating recommendation engine for new cost data --- backend/app/db/functions/recommendations_functions.py | 6 +++--- backend/app/plan/router.py | 1 - recommendations/VentilationRecommendations.py | 2 +- recommendations/optimiser/optimiser_functions.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 0bdf69ce..5d468db9 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -75,7 +75,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "property_id": property_id, "type": rec["type"], "description": rec["description"], - "estimated_cost": rec["cost"], + "estimated_cost": rec["total"], "default": rec["default"], "starting_u_value": rec.get("starting_u_value"), "new_u_value": rec.get("new_u_value"), @@ -102,10 +102,10 @@ def upload_recommendations(session: Session, recommendations_to_upload, property { "recommendation_id": recommendation_id, "material_id": part["id"], - "depth": part["depths"][0] if part["depths"] else None, + "depth": int(part["depth"]) if part["depth"] else None, "quantity": part["quantity"], "quantity_unit": part["quantity_unit"], - "estimated_cost": part["estimated_cost"], + "estimated_cost": part["total"], } for rec, recommendation_id in zip(recommendations_to_upload, uploaded_recommendation_ids) for part in rec["parts"] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fffc604e..37f673b4 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -318,7 +318,6 @@ async def trigger_plan(body: PlanTriggerRequest): batch_properties = input_properties[i:i + BATCH_SIZE] for p in batch_properties: - # Your existing operations property_details_epc = p.get_property_details_epc( portfolio_id=body.portfolio_id, rating_lookup=rating_lookup diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index a0b188f7..c330104f 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -52,7 +52,7 @@ class VentilationRecommendations(Definitions): estimated_cost = n_units * part[0]["cost"] - part[0]["estimated_cost"] = estimated_cost + part[0]["total"] = estimated_cost part[0]["quantity"] = n_units part[0]["quantity_unit"] = "part" diff --git a/recommendations/optimiser/optimiser_functions.py b/recommendations/optimiser/optimiser_functions.py index 869880cf..03aa38bd 100644 --- a/recommendations/optimiser/optimiser_functions.py +++ b/recommendations/optimiser/optimiser_functions.py @@ -22,7 +22,7 @@ def prepare_input_measures(property_recommendations, goal): [ { "id": rec["recommendation_id"], - "cost": rec["cost"], + "cost": rec["total"], "gain": rec[goal_key], "type": rec["type"] } From d942eacc557a032b84c7e9b67c4c55a80028a627 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 24 Nov 2023 11:54:46 +0000 Subject: [PATCH 27/27] finished updating recommendations for now to build portfolios --- backend/app/db/functions/portfolio_functions.py | 2 ++ backend/app/db/functions/recommendations_functions.py | 3 ++- backend/app/plan/router.py | 2 +- recommendations/FireplaceRecommendations.py | 2 ++ recommendations/VentilationRecommendations.py | 4 +++- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/app/db/functions/portfolio_functions.py b/backend/app/db/functions/portfolio_functions.py index 37b6bf37..08e15a32 100644 --- a/backend/app/db/functions/portfolio_functions.py +++ b/backend/app/db/functions/portfolio_functions.py @@ -8,6 +8,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int): aggregates = ( session.query( func.sum(Recommendation.estimated_cost).label("cost"), + func.sum(Recommendation.total_work_hours).label("total_work_hours"), # For future usage we will aggregate multiple fields in this step # func.sum(Recommendation.heat_demand).label("total_heat_demand"), # func.sum(Recommendation.energy_savings).label("total_energy_savings") @@ -20,6 +21,7 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int): aggregates_dict = { "cost": aggregates.cost or 0, + "total_work_hours": aggregates.total_work_hours or 0, # "total_heat_demand": aggregates.total_heat_demand or 0, # "total_energy_savings": aggregates.total_energy_savings or 0 } diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 5d468db9..34c4ef96 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -79,7 +79,8 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "default": rec["default"], "starting_u_value": rec.get("starting_u_value"), "new_u_value": rec.get("new_u_value"), - "sap_points": rec["sap_points"] + "sap_points": rec["sap_points"], + "total_work_hours": rec["labour_hours"], } for rec in recommendations_to_upload ] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 37f673b4..a20369cc 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -367,7 +367,7 @@ async def trigger_plan(body: PlanTriggerRequest): # the portfolion level impact aggregate_portfolio_recommendations(session, portfolio_id=body.portfolio_id) - # Commit all changes at once + # Commit final changes session.commit() except IntegrityError: logger.error("Database integrity error occurred", exc_info=True) diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 9524c75a..30ab1ad2 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -44,5 +44,7 @@ class FireplaceRecommendations(Definitions): "new_u_value": None, "sap_points": None, "total": estimated_cost, + # Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal + "labour_hours": 6 * number_open_fireplaces } ] diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index c330104f..419029a3 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -61,10 +61,12 @@ class VentilationRecommendations(Definitions): { "parts": part, "type": part[0]["type"], - "description": "Install %s" % part[0]["description"], + "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, "sap_points": None, "total": estimated_cost, + # We use a very simple and rough estimate of 4 hours per unit + "labour_hours": 4 * n_units } ]